summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.github/workflows/integration.yml40
-rw-r--r--.github/workflows/translations-update.yml56
-rw-r--r--.weblate3
-rw-r--r--Makefile2
-rw-r--r--README.rst2
-rw-r--r--docs/dev/translation.rst98
-rw-r--r--docs/dev/translation.svg45
-rwxr-xr-xmanage202
-rw-r--r--requirements-dev.txt1
9 files changed, 351 insertions, 98 deletions
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
index 78fc66e3f..84ce51a04 100644
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -91,7 +91,7 @@ jobs:
COMMIT_MESSAGE: build from commit ${{ github.sha }}
babel:
- name: Babel
+ name: Update translations branch
runs-on: ubuntu-20.04
if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }}
needs:
@@ -102,32 +102,32 @@ jobs:
- name: Checkout
uses: actions/checkout@v2
with:
- persist-credentials: false
+ fetch-depth: '0'
+ token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.9'
architecture: 'x64'
+ - name: Cache Python dependencies
+ id: cache-python
+ uses: actions/cache@v2
+ with:
+ path: ./local
+ key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}
+ - name: weblate & git setup
+ env:
+ WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }}
+ run: |
+ mkdir -p ~/.config
+ echo "${WEBLATE_CONFIG}" > ~/.config/weblate
+ git config --global user.email "searxng-bot@users.noreply.github.com"
+ git config --global user.name "searxng-bot"
- name: Update transations
id: update
- continue-on-error: true
- run: make V=1 ci.babel.update
- - name: Open pull request
- if: steps.update.outcome == 'success'
- uses: peter-evans/create-pull-request@v3
- with:
- commit-message: Update translations (pot, po)
- committer: searx-bot <noreply@github.com>
- author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
- signoff: false
- branch: update_translations_pot
- delete-branch: true
- draft: false
- title: 'Update translations (pot, po)'
- body: |
- Update messages.pot and messages.po files
- labels: |
- translation
+ run: |
+ git restore utils/brand.env
+ make V=1 weblate.push.translations
dockers:
name: Docker
diff --git a/.github/workflows/translations-update.yml b/.github/workflows/translations-update.yml
new file mode 100644
index 000000000..c3120ce1e
--- /dev/null
+++ b/.github/workflows/translations-update.yml
@@ -0,0 +1,56 @@
+name: "Update translations"
+on:
+ schedule:
+ - cron: "05 07 * * 5"
+ workflow_dispatch:
+
+jobs:
+ babel:
+ name: "create PR for additons from weblate"
+ runs-on: ubuntu-20.04
+ if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }}
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v2
+ with:
+ fetch-depth: '0'
+ token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
+ - name: Set up Python
+ uses: actions/setup-python@v2
+ with:
+ python-version: '3.9'
+ architecture: 'x64'
+ - name: Cache Python dependencies
+ id: cache-python
+ uses: actions/cache@v2
+ with:
+ path: ./local
+ key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}
+ - name: weblate & git setup
+ env:
+ WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }}
+ run: |
+ mkdir -p ~/.config
+ echo "${WEBLATE_CONFIG}" > ~/.config/weblate
+ git config --global user.email "searxng-bot@users.noreply.github.com"
+ git config --global user.name "searxng-bot"
+ - name: Merge and push transation updates
+ run: |
+ make V=1 weblate.translations.commit
+ - name: Create Pull Request
+ id: cpr
+ uses: peter-evans/create-pull-request@v3
+ with:
+ token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
+ commit-message: Update translations
+ committer: searxng-bot <searxng-bot@users.noreply.github.com>
+ author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
+ signoff: false
+ branch: translations_update
+ delete-branch: true
+ draft: false
+ title: 'Update translations'
+ body: |
+ Update translations
+ labels: |
+ translation
diff --git a/.weblate b/.weblate
new file mode 100644
index 000000000..80bdba884
--- /dev/null
+++ b/.weblate
@@ -0,0 +1,3 @@
+[weblate]
+url = https://weblate.bubu1.eu/api/
+translation = searxng/searxng
diff --git a/Makefile b/Makefile
index d48f5c531..88edef5cf 100644
--- a/Makefile
+++ b/Makefile
@@ -77,7 +77,7 @@ test.shell:
# wrap ./manage script
MANAGE += buildenv
-MANAGE += ci.babel.update babel.extract babel.update babel.compile
+MANAGE += weblate.translations.commit weblate.push.translations
MANAGE += data.all data.languages data.useragents data.osm_keys_tags
MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean
MANAGE += docker.build docker.push docker.buildx
diff --git a/README.rst b/README.rst
index 1ab7ee201..a4d2a30ef 100644
--- a/README.rst
+++ b/README.rst
@@ -32,6 +32,8 @@ Privacy-respecting, hackable `metasearch engine`_
.. |commits| image:: https://img.shields.io/github/commit-activity/y/searxng/searxng?color=yellow&label=commits
:target: https://github.com/searxng/searxng/commits/master
+.. |weblate| image:: https://weblate.bubu1.eu/widgets/searxng/-/searxng/svg-badge.svg
+ :target: https://weblate.bubu1.eu/projects/searxng/
If you are looking for running instances, ready to use, then visit searx.space_.
diff --git a/docs/dev/translation.rst b/docs/dev/translation.rst
index 523dcf78c..922c29c54 100644
--- a/docs/dev/translation.rst
+++ b/docs/dev/translation.rst
@@ -4,69 +4,77 @@
Translation
===========
-.. _searx@transifex: https://www.transifex.com/asciimoo/searx/
+.. _weblate.bubu1.eu: https://weblate.bubu1.eu/projects/searxng/
+.. _Weblate: https://docs.weblate.org
+.. _translations branch: https://github.com/searxng/searxng/tree/translations
+.. _orphan branch: https://git-scm.com/docs/git-checkout#Documentation/git-checkout.txt---orphanltnewbranchgt
+.. _Weblate repository: https://weblate.bubu1.eu/projects/searxng/searxng/#repository
+.. _wlc: https://docs.weblate.org/en/latest/wlc.html
-Translation currently takes place on `searx@transifex`_
+.. |translated| image:: https://weblate.bubu1.eu/widgets/searxng/-/searxng/svg-badge.svg
+ :target: https://weblate.bubu1.eu/projects/searxng/
-Requirements
-============
+.. sidebar:: |translated|
-* Transifex account
+ - Weblate_
+ - SearXNG `translations branch`_
+ - SearXNG `Weblate repository`_
+ - Weblate Client: wlc_
+ - Babel Command-Line: `pybabel <http://babel.pocoo.org/en/latest/cmdline.html>`_
+ - `weblate workflow <https://docs.weblate.org/en/latest/workflows.html>`_
-Init Transifex project
-======================
+Translation takes place on weblate.bubu1.eu_.
-After installing ``transifex`` using pip, run the following command to
-initialize the project.
+Translations which has been added by translators on the weblate.bubu1.eu_ UI are
+committed to Weblate's counterpart of the SearXNG *origin* repository which is
+located at ``https://weblate.bubu1.eu/git/searxng/searxng``.
-.. code:: sh
+There is no need to clone this repository, :ref:`SearXNG Weblate workflow` take
+care of the synchronization with the *origin*. To avoid merging commits from
+the counterpart directly on the ``master`` branch of *SearXNG origin*, a *pull
+request* (PR) is created by this workflow.
- ./manage pyenv.cmd tx init # Transifex instance: https://www.transifex.com/asciimoo/searx/
+Weblate monitors the `translations branch`_, not the ``master`` branch. This
+branch is an `orphan branch`_, decoupled from the master branch (we already know
+orphan branches from the ``gh-pages``). The `translations branch`_ contains
+only the
+- ``translation/messages.pot`` and the
+- ``translation/*/messages.po`` files, nothing else.
-After ``$HOME/.transifexrc`` is created, get a Transifex API key and insert it
-into the configuration file.
-Create a configuration file for ``tx`` named ``$HOME/.tx/config``.
+.. _SearXNG Weblate workflow:
-.. code:: ini
+.. figure:: translation.svg
- [main]
- host = https://www.transifex.com
- [searx.messagespo]
- file_filter = searx/translations/<lang>/LC_MESSAGES/messages.po
- source_file = messages.pot
- source_lang = en
- type = PO
+ SearXNG's PR workflow to be in sync with Weblate
+Sync from *origin* to *weblate*: using ``make weblate.push.translations``
+ For each commit on the ``master`` branch of SearXNG *origin* the GitHub job
+ :origin:`babel / Update translations branch
+ <.github/workflows/integration.yml>` checks for updated translations.
-Then run ``tx set``:
+Sync from *weblate* to *origin*: using ``make weblate.translations.commit``
+ Every Friday, the GitHub workflow :origin:`babel / create PR for additons from
+ weblate <.github/workflows/translations-update.yml>` creates a PR with the
+ updated translation files:
-.. code:: shell
+ - ``translation/messages.pot``,
+ - ``translation/*/messages.po`` and
+ - ``translation/*/messages.mo``
- ./manage pyenv.cmd tx set --auto-local \
- -r searx.messagespo 'searx/translations/<lang>/LC_MESSAGES/messages.po' \
- --source-lang en --type PO --source-file messages.pot --execute
+wlc
+===
+.. _wlc configuration: https://docs.weblate.org/en/latest/wlc.html#wlc-config
+.. _API key: https://weblate.bubu1.eu/accounts/profile/#api
-Update translations
-===================
+All weblate integration is done by GitHub workflows, but if you want to use wlc_,
+copy this content into `wlc configuration`_ in your HOME ``~/.config/weblate``
-To retrieve the latest translations, pull it from Transifex.
+.. code-block:: ini
-.. code:: sh
+ [keys]
+ https://weblate.bubu1.eu/api/ = APIKEY
- ./manage pyenv.cmd tx pull -a
- [?] Enter your api token: ....
-
-Then check the new languages. If strings translated are not enough, delete those
-folders, because those should not be compiled. Call the command below to compile
-the ``.po`` files.
-
-.. code:: shell
-
- ./manage pyenv.cmd pybabel compile -d searx/translations
-
-
-After the compilation is finished commit the ``.po`` and ``.mo`` files and
-create a PR.
+Replace ``APIKEY`` by your `API key`_.
diff --git a/docs/dev/translation.svg b/docs/dev/translation.svg
new file mode 100644
index 000000000..70400cca7
--- /dev/null
+++ b/docs/dev/translation.svg
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" contentScriptType="application/ecmascript" contentStyleType="text/css" height="968px" preserveAspectRatio="none" style="width:1008px;height:968px;background:#FFFFFF;" version="1.1" viewBox="0 0 1008 968" width="1008px" zoomAndPan="magnify"><defs><filter height="300%" id="fwj4fqzvak25j" width="300%" x="-1" y="-1"><feGaussianBlur result="blurOut" stdDeviation="2.0"/><feColorMatrix in="blurOut" result="blurOut2" type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 .4 0"/><feOffset dx="4.0" dy="4.0" in="blurOut2" result="blurOut3"/><feBlend in="SourceGraphic" in2="blurOut3" mode="normal"/></filter></defs><g><rect fill="#FFFFFF" filter="url(#fwj4fqzvak25j)" height="320.7266" style="stroke:#000000;stroke-width:2.0;" width="981" x="10" y="116.7266"/><rect fill="#FFFFFF" filter="url(#fwj4fqzvak25j)" height="289.5938" style="stroke:#000000;stroke-width:2.0;" width="961" x="20" y="140.8594"/><rect fill="#FFFFFF" filter="url(#fwj4fqzvak25j)" height="324.7266" style="stroke:#000000;stroke-width:2.0;" width="848" x="143" y="494.5859"/><rect fill="#FFFFFF" filter="url(#fwj4fqzvak25j)" height="245.3281" style="stroke:#000000;stroke-width:2.0;" width="828" x="153" y="518.7188"/><line style="stroke:#A80036;stroke-width:1.0;stroke-dasharray:5.0,5.0;" x1="91" x2="91" y1="56.5938" y2="908.5781"/><line style="stroke:#A80036;stroke-width:1.0;stroke-dasharray:5.0,5.0;" x1="270" x2="270" y1="56.5938" y2="908.5781"/><line style="stroke:#A80036;stroke-width:1.0;stroke-dasharray:5.0,5.0;" x1="514" x2="514" y1="56.5938" y2="908.5781"/><line style="stroke:#A80036;stroke-width:1.0;stroke-dasharray:5.0,5.0;" x1="738" x2="738" y1="56.5938" y2="908.5781"/><line style="stroke:#A80036;stroke-width:1.0;stroke-dasharray:5.0,5.0;" x1="899" x2="899" y1="56.5938" y2="908.5781"/><rect fill="#FEFECE" filter="url(#fwj4fqzvak25j)" height="30.2969" style="stroke:#A80036;stroke-width:1.5;" width="119" x="30" y="21.2969"/><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="105" x="37" y="41.292">master branch</text><rect fill="#FEFECE" filter="url(#fwj4fqzvak25j)" height="30.2969" style="stroke:#A80036;stroke-width:1.5;" width="119" x="30" y="907.5781"/><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="105" x="37" y="927.5732">master branch</text><rect fill="#FEFECE" filter="url(#fwj4fqzvak25j)" height="30.2969" style="stroke:#A80036;stroke-width:1.5;" width="211" x="163" y="21.2969"/><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="197" x="170" y="41.292">translations_update branch</text><rect fill="#FEFECE" filter="url(#fwj4fqzvak25j)" height="30.2969" style="stroke:#A80036;stroke-width:1.5;" width="211" x="163" y="907.5781"/><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="197" x="170" y="927.5732">translations_update branch</text><rect fill="#FEFECE" filter="url(#fwj4fqzvak25j)" height="30.2969" style="stroke:#A80036;stroke-width:1.5;" width="153" x="436" y="21.2969"/><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="139" x="443" y="41.292">translations branch</text><rect fill="#FEFECE" filter="url(#fwj4fqzvak25j)" height="30.2969" style="stroke:#A80036;stroke-width:1.5;" width="153" x="436" y="907.5781"/><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="139" x="443" y="927.5732">translations branch</text><rect fill="#FEFECE" filter="url(#fwj4fqzvak25j)" height="46.5938" style="stroke:#A80036;stroke-width:1.5;" width="153" x="660" y="5"/><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="120" x="676.5" y="24.9951">weblate clone of</text><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="139" x="667" y="41.292">SearXNG repository</text><rect fill="#FEFECE" filter="url(#fwj4fqzvak25j)" height="46.5938" style="stroke:#A80036;stroke-width:1.5;" width="153" x="660" y="907.5781"/><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="120" x="676.5" y="927.5732">weblate clone of</text><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="139" x="667" y="943.8701">SearXNG repository</text><rect fill="#FEFECE" filter="url(#fwj4fqzvak25j)" height="46.5938" style="stroke:#A80036;stroke-width:1.5;" width="140" x="827" y="5"/><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="58" x="868" y="24.9951">weblate</text><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="122" x="834" y="41.292">pending changes</text><rect fill="#FEFECE" filter="url(#fwj4fqzvak25j)" height="46.5938" style="stroke:#A80036;stroke-width:1.5;" width="140" x="827" y="907.5781"/><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="58" x="868" y="927.5732">weblate</text><text fill="#000000" font-family="sans-serif" font-size="14" lengthAdjust="spacing" textLength="122" x="834" y="943.8701">pending changes</text><rect fill="#EEEEEE" filter="url(#fwj4fqzvak25j)" height="3" style="stroke:#EEEEEE;stroke-width:1.0;" width="1001" x="0" y="87.1602"/><line style="stroke:#000000;stroke-width:1.0;" x1="0" x2="1001" y1="87.1602" y2="87.1602"/><line style="stroke:#000000;stroke-width:1.0;" x1="0" x2="1001" y1="90.1602" y2="90.1602"/><rect fill="#EEEEEE" filter="url(#fwj4fqzvak25j)" height="23.1328" style="stroke:#000000;stroke-width:2.0;" width="230" x="385.5" y="76.5938"/><text fill="#000000" font-family="sans-serif" font-size="13" font-weight="bold" lengthAdjust="spacing" textLength="211" x="391.5" y="92.6606">for each commit on master</text><path d="M10,116.7266 L314,116.7266 L314,123.7266 L304,133.7266 L10,133.7266 L10,116.7266 " fill="#EEEEEE" style="stroke:#000000;stroke-width:1.0;"/><rect fill="none" height="320.7266" style="stroke:#000000;stroke-width:2.0;" width="981" x="10" y="116.7266"/><text fill="#000000" font-family="sans-serif" font-size="13" font-weight="bold" lengthAdjust="spacing" textLength="259" x="25" y="129.7935">.github/workflows/integration.yml</text><path d="M20,140.8594 L332,140.8594 L332,147.8594 L322,157.8594 L20,157.8594 L20,140.8594 " fill="#EEEEEE" style="stroke:#000000;stroke-width:1.0;"/><rect fill="none" height="289.5938" style="stroke:#000000;stroke-width:2.0;" width="961" x="20" y="140.8594"/><text fill="#000000" font-family="sans-serif" font-size="13" font-weight="bold" lengthAdjust="spacing" textLength="267" x="35" y="153.9263">make weblate.push.translations</text><polygon fill="#FBFB77" filter="url(#fwj4fqzvak25j)" points="711,162.9922,765,162.9922,775,173.9922,765,185.9922,711,185.9922,701,173.9922,711,162.9922" style="stroke:#A80036;stroke-width:1.0;"/><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="50" x="713" y="179.0591">wlc lock</text><polygon fill="#A80036" points="726.5,208.2578,736.5,212.2578,726.5,216.2578,730.5,212.2578" style="stroke:#A80036;stroke-width:1.0;"/><line style="stroke:#A80036;stroke-width:1.0;" x1="514.5" x2="732.5" y1="212.2578" y2="212.2578"/><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="47" x="521.5" y="207.1919">wlc pull</text><polygon fill="#A80036" points="749.5,237.3906,739.5,241.3906,749.5,245.3906,745.5,241.3906" style="stroke:#A80036;stroke-width:1.0;"/><line style="stroke:#A80036;stroke-width:1.0;" x1="743.5" x2="898" y1="241.3906" y2="241.3906"/><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="74" x="755.5" y="236.3247">wlc commit</text><polygon fill="#A80036" points="525.5,266.5234,515.5,270.5234,525.5,274.5234,521.5,270.5234" style="stroke:#A80036;stroke-width:1.0;"/><line style="stroke:#A80036;stroke-width:1.0;" x1="519.5" x2="737.5" y1="270.5234" y2="270.5234"/><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="196" x="531.5" y="265.4575">git merge weblate/translations</text><polygon fill="#A80036" points="502.5,310.7891,512.5,314.7891,502.5,318.7891,506.5,314.7891" style="stroke:#A80036;stroke-width:1.0;"/><line style="stroke:#A80036;stroke-width:1.0;" x1="91.5" x2="508.5" y1="314.7891" y2="314.7891"/><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="101" x="98.5" y="294.5903">pybabel extract</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="399" x="98.5" y="309.7231">extract messages, store messages.pot on translations branch</text><polygon fill="#A80036" points="726.5,385.3203,736.5,389.3203,726.5,393.3203,730.5,389.3203" style="stroke:#A80036;stroke-width:1.0;"/><line style="stroke:#A80036;stroke-width:1.0;" x1="514.5" x2="732.5" y1="389.3203" y2="389.3203"/><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="200" x="521.5" y="338.856">pybabel update (messages.po)</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="163" x="521.5" y="353.9888">git add searx/translations</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="69" x="521.5" y="369.1216">git commit</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="51" x="521.5" y="384.2544">git push</text><polygon fill="#FBFB77" filter="url(#fwj4fqzvak25j)" points="703,402.3203,773,402.3203,783,413.3203,773,425.3203,703,425.3203,693,413.3203,703,402.3203" style="stroke:#A80036;stroke-width:1.0;"/><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="66" x="705" y="418.3872">wlc unlock</text><rect fill="#EEEEEE" filter="url(#fwj4fqzvak25j)" height="3" style="stroke:#EEEEEE;stroke-width:1.0;" width="1001" x="0" y="465.0195"/><line style="stroke:#000000;stroke-width:1.0;" x1="0" x2="1001" y1="465.0195" y2="465.0195"/><line style="stroke:#000000;stroke-width:1.0;" x1="0" x2="1001" y1="468.0195" y2="468.0195"/><rect fill="#EEEEEE" filter="url(#fwj4fqzvak25j)" height="23.1328" style="stroke:#000000;stroke-width:2.0;" width="111" x="445" y="454.4531"/><text fill="#000000" font-family="sans-serif" font-size="13" font-weight="bold" lengthAdjust="spacing" textLength="92" x="451" y="470.52">every Friday</text><path d="M143,494.5859 L513,494.5859 L513,501.5859 L503,511.5859 L143,511.5859 L143,494.5859 " fill="#EEEEEE" style="stroke:#000000;stroke-width:1.0;"/><rect fill="none" height="324.7266" style="stroke:#000000;stroke-width:2.0;" width="848" x="143" y="494.5859"/><text fill="#000000" font-family="sans-serif" font-size="13" font-weight="bold" lengthAdjust="spacing" textLength="325" x="158" y="507.6528">.github/workflows/translations-update.yml</text><path d="M153,518.7188 L465,518.7188 L465,525.7188 L455,535.7188 L153,535.7188 L153,518.7188 " fill="#EEEEEE" style="stroke:#000000;stroke-width:1.0;"/><rect fill="none" height="245.3281" style="stroke:#000000;stroke-width:2.0;" width="828" x="153" y="518.7188"/><text fill="#000000" font-family="sans-serif" font-size="13" font-weight="bold" lengthAdjust="spacing" textLength="267" x="168" y="531.7856">make weblate.translations.commit</text><polygon fill="#FBFB77" filter="url(#fwj4fqzvak25j)" points="711,540.8516,765,540.8516,775,551.8516,765,563.8516,711,563.8516,701,551.8516,711,540.8516" style="stroke:#A80036;stroke-width:1.0;"/><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="50" x="713" y="556.9185">wlc lock</text><polygon fill="#A80036" points="726.5,586.1172,736.5,590.1172,726.5,594.1172,730.5,590.1172" style="stroke:#A80036;stroke-width:1.0;"/><line style="stroke:#A80036;stroke-width:1.0;" x1="514.5" x2="732.5" y1="590.1172" y2="590.1172"/><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="47" x="521.5" y="585.0513">wlc pull</text><polygon fill="#A80036" points="749.5,615.25,739.5,619.25,749.5,623.25,745.5,619.25" style="stroke:#A80036;stroke-width:1.0;"/><line style="stroke:#A80036;stroke-width:1.0;" x1="743.5" x2="898" y1="619.25" y2="619.25"/><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="74" x="755.5" y="614.1841">wlc commit</text><polygon fill="#A80036" points="525.5,644.3828,515.5,648.3828,525.5,652.3828,521.5,648.3828" style="stroke:#A80036;stroke-width:1.0;"/><line style="stroke:#A80036;stroke-width:1.0;" x1="519.5" x2="737.5" y1="648.3828" y2="648.3828"/><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="196" x="531.5" y="643.3169">git merge weblate/translations</text><polygon fill="#A80036" points="281.5,718.9141,271.5,722.9141,281.5,726.9141,277.5,722.9141" style="stroke:#A80036;stroke-width:1.0;"/><line style="stroke:#A80036;stroke-width:1.0;" x1="275.5" x2="513.5" y1="722.9141" y2="722.9141"/><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="105" x="287.5" y="672.4497">pybabel compile</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="134" x="287.5" y="687.5825">cp searx/translations</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="44" x="287.5" y="702.7153">git add</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="69" x="287.5" y="717.8481">git commit</text><polygon fill="#FBFB77" filter="url(#fwj4fqzvak25j)" points="703,735.9141,773,735.9141,783,746.9141,773,758.9141,703,758.9141,693,746.9141,703,735.9141" style="stroke:#A80036;stroke-width:1.0;"/><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="66" x="705" y="751.981">wlc unlock</text><polygon fill="#FBFB77" filter="url(#fwj4fqzvak25j)" points="175,776.0469,365,776.0469,375,795.0469,365,814.0469,175,814.0469,165,795.0469,175,776.0469" style="stroke:#A80036;stroke-width:1.0;"/><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="186" x="177" y="792.1138">create or update pull request</text><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="135" x="177" y="807.2466">"Update translations"</text><rect fill="#EEEEEE" filter="url(#fwj4fqzvak25j)" height="3" style="stroke:#EEEEEE;stroke-width:1.0;" width="1001" x="0" y="846.8789"/><line style="stroke:#000000;stroke-width:1.0;" x1="0" x2="1001" y1="846.8789" y2="846.8789"/><line style="stroke:#000000;stroke-width:1.0;" x1="0" x2="1001" y1="849.8789" y2="849.8789"/><rect fill="#EEEEEE" filter="url(#fwj4fqzvak25j)" height="23.1328" style="stroke:#000000;stroke-width:2.0;" width="168" x="416.5" y="836.3125"/><text fill="#000000" font-family="sans-serif" font-size="13" font-weight="bold" lengthAdjust="spacing" textLength="149" x="422.5" y="852.3794">developper's review</text><polygon fill="#A80036" points="102.5,886.5781,92.5,890.5781,102.5,894.5781,98.5,890.5781" style="stroke:#A80036;stroke-width:1.0;"/><line style="stroke:#A80036;stroke-width:1.0;" x1="96.5" x2="269.5" y1="890.5781" y2="890.5781"/><text fill="#000000" font-family="sans-serif" font-size="13" lengthAdjust="spacing" textLength="121" x="108.5" y="885.5122">merge pull request</text><!--MD5=[e3d4a82efdc738a59c5df45b59de0f06]
+@startuml
+participant "master branch" as master
+participant "translations_update branch" as translations_update
+participant "translations branch" as translations
+participant "weblate clone of\ntranslations branch" as weblate
+participant "weblate\npending changes " as weblate_change
+
+== for each commit on master ==
+
+group .github/workflows/integration.yml
+ group make weblate.push.translations
+ hnote over weblate : wlc lock
+ translations -> weblate: wlc pull
+ weblate_change -> weblate: wlc commit
+ weblate -> translations: git merge weblate/translations
+ master -> translations: pybabel extract\nextract messages, store messages.pot on translations branch
+ translations -> weblate: pybabel update (messages.po)\ngit add searx/translations\ngit commit\ngit push
+ hnote over weblate : wlc unlock
+ end
+end
+
+== every Friday ==
+
+group .github/workflows/translations-update.yml
+ group make weblate.translations.commit
+ hnote over weblate : wlc lock
+ translations -> weblate: wlc pull
+ weblate_change -> weblate: wlc commit
+ weblate -> translations: git merge weblate/translations
+ translations -> translations_update: pybabel compile\ncp searx/translations\ngit add\ngit commit
+ hnote over weblate : wlc unlock
+ end
+ hnote over translations_update : create or update pull request\n"Update translations"
+end
+
+== developper's review ==
+
+translations_update -> master: merge pull request
+@enduml
+
+See
+https://plantuml.com/en/sequence-diagram
+https://www.planttext.com
+--></g></svg>
diff --git a/manage b/manage
index 42c617f5b..a855512f5 100755
--- a/manage
+++ b/manage
@@ -45,9 +45,8 @@ help() {
buildenv:
rebuild ./utils/brand.env
babel.:
- extract : extract messages from source files and generate POT file
- update : update existing message catalogs from POT file
- compile : compile translation catalogs into binary MO files
+ master.to.translations: update the translations branch from the messages of the master branch.
+ translations.to.master: copy change from the translations branch to the master branch.
data.:
all : update searx/languages.py and ./data/*
languages : update searx/data/engines_languages.json & searx/languages.py
@@ -122,46 +121,185 @@ buildenv() {
return "${PIPESTATUS[0]}"
}
-babel.sha256sum() {
- grep "msgid" "searx/translations/messages.pot" | sort | sha256sum | cut -f1 -d ' '
-}
+TRANSLATIONS_WORKTREE="$CACHE/translations"
+
+weblate.translations.worktree() {
+
+ # Create git worktree ${TRANSLATIONS_WORKTREE} and checkout branch
+ # 'translations' from Weblate's counterpart (weblate) of the SearXNG
+ # (origin).
+ #
+ # remote weblate https://weblate.bubu1.eu/git/searxng/searxng/
-ci.babel.update() {
- local sha_before
( set -e
- sha_before="$(babel.sha256sum)"
- babel.extract
- if [ "$(babel.sha256sum)" = "${sha_before}" ]; then
- build_msg BABEL 'no changes detected, exiting'
- return 1
+ if ! git remote get-url weblate 2> /dev/null; then
+ git remote add weblate https://weblate.bubu1.eu/git/searxng/searxng/
+ fi
+ if [ -d "${TRANSLATIONS_WORKTREE}" ]; then
+ pushd .
+ cd "${TRANSLATIONS_WORKTREE}"
+ git reset --hard HEAD
+ git pull origin translations
+ popd
+ else
+ mkdir -p "${TRANSLATIONS_WORKTREE}"
+ git worktree add "${TRANSLATIONS_WORKTREE}" translations
fi
- babel.update
- build_msg BABEL 'update done, edit .po files if required and run babel.compile'
)
- dump_return $?
}
-babel.extract() {
- build_msg BABEL 'extract messages from source files and generate POT file'
- pyenv.cmd pybabel extract -F babel.cfg \
- -o "searx/translations/messages.pot" \
- "searx/"
+weblate.to.translations() {
+
+ # Update 'translations' branch of SearXNG (origin) with last additions from
+ # Weblate.
+
+ # 1. Check if Weblate is locked, if not die with error message
+ # 2. On Weblate's counterpart (weblate), pull master and translations branch
+ # from SearXNG (origin).
+ # 3. Commit changes made in a Weblate object on Weblate's counterpart
+ # (weblate).
+ # 4. In translations worktree, merge changes of branch 'translations' from
+ # remote 'weblate' and push it on branch 'translations' of 'origin'
+
+ ( set -e
+ if [ "$(pyenv.cmd wlc lock-status)" != "locked: True" ]; then
+ die 1 "weblate must be locked, currently: $(pyenv.cmd wlc lock-status)"
+ fi
+ # weblate: commit pending changes
+ pyenv.cmd wlc pull
+ pyenv.cmd wlc commit
+
+ # get the translations in a worktree
+ weblate.translations.worktree
+
+ pushd "${TRANSLATIONS_WORKTREE}"
+ git remote update weblate
+ git merge weblate/translations
+ git push
+ popd
+ )
dump_return $?
}
-babel.update() {
- build_msg BABEL 'update existing message catalogs from POT file'
- pyenv.cmd pybabel update -N \
- -i "searx/translations/messages.pot" \
- -d "searx/translations"
- dump_return $?
+weblate.translations.commit() {
+
+ # Update 'translations' branch of SearXNG (origin) with last additions from
+ # Weblate. Copy the changes to the master branch, compile translations and
+ # create a commit in the local branch (master)
+
+ local existing_commit_hash commit_body commit_message exitcode
+ ( set -e
+ # lock change on weblate
+ pyenv.cmd wlc lock
+
+ # get translations branch in git worktree (TRANSLATIONS_WORKTREE)
+ weblate.translations.worktree
+ existing_commit_hash=$(cd "${TRANSLATIONS_WORKTREE}"; git log -n1 --pretty=format:'%h')
+
+ # pull weblate commits
+ weblate.to.translations
+
+ # copy the changes to the master branch
+ cp -rv --preserve=mode,timestamps "${TRANSLATIONS_WORKTREE}/searx/translations" "searx"
+
+ # compile translations
+ build_msg BABEL 'compile translation catalogs into binary MO files'
+ pyenv.cmd pybabel compile --statistics \
+ -d "searx/translations"
+ # git add/commit (no push)
+ commit_body=$(cd "${TRANSLATIONS_WORKTREE}"; git log --pretty=format:'%h - %as - %aN <%ae>' "${existing_commit_hash}..HEAD")
+ commit_message=$(echo -e "[translations] update\n${commit_body}")
+ git add searx/translations
+ git commit -m "${commit_message}"
+ )
+ exitcode=$?
+ ( # make sure to always unlock weblate
+ set -e
+ pyenv.cmd wlc unlock
+ )
+ dump_return $exitcode
}
-babel.compile() {
- build_msg BABEL 'compile translation catalogs into binary MO files'
- pyenv.cmd pybabel compile --statistics \
- -d "searx/translations"
- dump_return $?
+weblate.push.translations() {
+
+ # Push *translation changes* from SearXNG (origin) to Weblate's counterpart
+ # (weblate).
+
+ # In branch master of SearXNG (origin) check for meaningful changes in
+ # folder 'searx/translations', commit changes on branch 'translations' and
+ # at least, pull updated branches on Weblate's counterpart (weblate).
+
+ # 1. Create git worktree ${TRANSLATIONS_WORKTREE} and checkout branch
+ # 'translations' from remote 'weblate'.
+ # 2. Stop if there is no meaningful change in the 'master' branch (origin),
+ # compared to the 'translations' branch (weblate), otherwise ...
+ # 3. Update 'translations' branch of SearXNG (origin) with last additions
+ # from Weblate.
+ # 5. Notify Weblate to pull updated 'master' & 'translations' branch.
+
+ local messages_pot diff_messages_pot last_commit_hash last_commit_detail \
+ last_commit_message exitcode
+ ( set -e
+ # lock change on weblate
+ pyenv.cmd wlc lock
+
+ # get translations branch in git worktree (TRANSLATIONS_WORKTREE)
+ weblate.translations.worktree
+
+ # update messages.pot in the master branch
+ build_msg BABEL 'extract messages from source files and generate POT file'
+ messages_pot="${TRANSLATIONS_WORKTREE}/searx/translations/messages.pot"
+ pyenv.cmd pybabel extract -F babel.cfg \
+ -o "${messages_pot}" \
+ "searx/"
+
+ # stop if there is no meaningful change in the master branch
+ diff_messages_pot=$(cd "${TRANSLATIONS_WORKTREE}";\
+ git diff -- "searx/translations/messages.pot")
+ if ! echo "$diff_messages_pot" | grep -qE "[\+\-](msgid|msgstr)"; then
+ build_msg BABEL 'no changes detected, exiting'
+ return 0
+ fi
+
+ # save messages.pot in the translations branch for later
+ pushd "${TRANSLATIONS_WORKTREE}"
+ git stash push
+ popd
+
+ # merge weblate commits into the translations branch
+ weblate.to.translations
+
+ # restore messages.pot in the translations branch
+ pushd "${TRANSLATIONS_WORKTREE}"
+ git stash pop
+ popd
+
+ # update messages.po files in the master branch
+ build_msg BABEL 'update existing message catalogs from POT file'
+ pyenv.cmd pybabel update -N \
+ -i "${messages_pot}" \
+ -d "${TRANSLATIONS_WORKTREE}/searx/translations"
+
+ # git add/commit/push
+ last_commit_hash=$(git log -n1 --pretty=format:'%h')
+ last_commit_detail=$(git log -n1 --pretty=format:'%h - %as - %aN <%ae>' "${last_commit_hash}")
+ last_commit_message="[translations] update messages.pot and messages.po files\nFrom ${last_commit_detail}"
+
+ pushd "${TRANSLATIONS_WORKTREE}"
+ git add searx/translations
+ git commit -m "${last_commit_message}"
+ git push
+ popd
+
+ # notify weblate to pull updated master & translations branch
+ pyenv.cmd wlc pull
+ )
+ exitcode=$?
+ ( # make sure to always unlock weblate
+ set -e
+ pyenv.cmd wlc unlock
+ )
+ dump_return $exitcode
}
data.all() {
diff --git a/requirements-dev.txt b/requirements-dev.txt
index a6f29db55..e8dc0221c 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -17,3 +17,4 @@ sphinx-autobuild==2021.3.14
linuxdoc==20210324
aiounittest==1.4.0
yamllint==1.26.2
+wlc==1.12
7apu2bq3rhoylwmucxizk54afw5jyda7pho4j5zdcqpwkufke7zdzwad.onion