diff options
| -rw-r--r-- | .github/workflows/data-update.yml | 67 | ||||
| -rw-r--r-- | utils/fetch_languages.py | 7 |
2 files changed, 71 insertions, 3 deletions
diff --git a/.github/workflows/data-update.yml b/.github/workflows/data-update.yml new file mode 100644 index 000000000..a97169767 --- /dev/null +++ b/.github/workflows/data-update.yml @@ -0,0 +1,67 @@ +name: "Update searx.data" +on: + schedule: + - cron: "37 13 * * 0" + +jobs: + updateData: + name: Update data + runs-on: ubuntu-20.04 + if: env.DATA_PR_TOKEN != null + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Install Ubuntu packages + run: | + sudo ./utils/searx.sh install packages + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.9' + architecture: 'x64' + + - name: Cache Python dependencies + id: cache-python + uses: actions/cache@v2 + with: + path: ./local + key: python-${{ matrix.os }}-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }} + + - name: Install Python dependencies + if: steps.cache-python.outputs.cache-hit != 'true' + run: | + make V=1 install + + - name: Fetch data + run: | + source local/py3/bin/activate + python utils/fetch_firefox_version.py + python utils/fetch_languages.py + python utils/fetch_ahmia_blacklist.py + python utils/fetch_wikidata_units.py + # python utils/fetch_currencies.py + + - name: Create Pull Request + id: cpr + uses: peter-evans/create-pull-request@v3 + with: + token: ${{ secrets.DATA_PR_TOKEN }} + commit-message: Update searx.data + committer: searx-bot <noreply@github.com> + author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> + signoff: false + branch: automatic-update-data + delete-branch: true + title: 'Update searx.data' + body: | + Update searx.data + labels: | + data + draft: false + + - name: Check outputs + run: | + echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" + echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" diff --git a/utils/fetch_languages.py b/utils/fetch_languages.py index 1f8dc7786..453693136 100644 --- a/utils/fetch_languages.py +++ b/utils/fetch_languages.py @@ -6,18 +6,19 @@ # are written in current directory to avoid overwriting in case something goes wrong. import json +from pathlib import Path from pprint import pformat from sys import path from babel import Locale, UnknownLocaleError from babel.languages import get_global path.append('../searx') # noqa -from searx import settings +from searx import settings, searx_dir from searx.engines import initialize_engines, engines # Output files. -engines_languages_file = 'engines_languages.json' -languages_file = 'languages.py' +engines_languages_file = Path(searx_dir) / 'data' / 'engines_languages.json' +languages_file = Path(searx_dir) / 'languages.py' # Fetchs supported languages for each engine and writes json file with those. |