Merge pull request #859 from return42/fix-814

[mod] add i18n infrastructure for SearXNG message files (searxng.msg)

8 files changed, 216 insertions, 98 deletions

diff --git a/babel.cfg b/babel.cfg
index 03bde6ca5..03913c256 100644
--- a/babel.cfg
+++ b/babel.cfg
@@ -1,4 +1,7 @@
+[extractors]
+searxng_msg = searx.babel_extract.extract
 [ignore: **/node_modules/**]
 [python: **.py]
 [jinja2: **/templates/**.html]
 extensions=jinja2.ext.autoescape,jinja2.ext.with_
+[searxng_msg: **/searxng.msg]
diff --git a/docs/dev/translation.rst b/docs/dev/translation.rst
index 922c29c54..9de9ff2de 100644
--- a/docs/dev/translation.rst
+++ b/docs/dev/translation.rst
@@ -16,6 +16,7 @@ Translation
 
 .. sidebar:: |translated|
 
+   - :ref:`searx.babel_extract`
    - Weblate_
    - SearXNG `translations branch`_
    - SearXNG `Weblate repository`_
diff --git a/docs/src/searx.babel_extract.rst b/docs/src/searx.babel_extract.rst
new file mode 100644
index 000000000..741d67fc1
--- /dev/null
+++ b/docs/src/searx.babel_extract.rst
@@ -0,0 +1,8 @@
+.. _searx.babel_extract:
+
+===============================
+Custom message extractor (i18n)
+===============================
+
+.. automodule:: searx.babel_extract
+  :members:
diff --git a/manage b/manage
index e70b7f4dd..9e5b59fc7 100755
--- a/manage
+++ b/manage
@@ -30,7 +30,7 @@ GECKODRIVER_VERSION="v0.30.0"
 export NODE_MINIMUM_VERSION="16.13.0"
 # SPHINXOPTS=
 BLACK_OPTIONS=("--target-version" "py37" "--line-length" "120" "--skip-string-normalization")
-BLACK_TARGETS=("--exclude" "searx/static,searx/languages.py" "searx" "searxng_extra" "tests")
+BLACK_TARGETS=("--exclude" "searx/static,searx/languages.py" "--include" "searxng.msg" "searx" "searxng_extra" "tests")
 
 pylint.FILES() {
 
@@ -41,6 +41,7 @@ pylint.FILES() {
     # These py files are linted by test.pylint()
 
     grep -l -r --include \*.py '^#[[:blank:]]*lint:[[:blank:]]*pylint' searx searxng_extra tests
+    find . -name searxng.msg
 }
 
 YAMLLINT_FILES=()
diff --git a/searx/babel_extract.py b/searx/babel_extract.py
new file mode 100644
index 000000000..5bcb1f0e9
--- /dev/null
+++ b/searx/babel_extract.py
@@ -0,0 +1,51 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# lint: pylint
+"""This module implements the :origin:`searxng_msg <babel.cfg>` extractor to
+extract messages from:
+
+- :origin:`searx/searxng.msg`
+
+The ``searxng.msg`` files are selected by Babel_, see Babel's configuration in
+:origin:`babel.cfg`::
+
+    searxng_msg = searx.babel_extract.extract
+    ...
+    [searxng_msg: **/searxng.msg]
+
+A ``searxng.msg`` file is a python file that is *executed* by the
+:py:obj:`extract` function.  Additional ``searxng.msg`` files can be added by:
+
+1. Adding a ``searxng.msg`` file in one of the SearXNG python packages and
+2. implement a method in :py:obj:`extract` that yields messages from this file.
+
+.. _Babel: https://babel.pocoo.org/en/latest/index.html
+
+"""
+
+from os import path
+
+SEARXNG_MSG_FILE = "searxng.msg"
+_MSG_FILES = [path.join(path.dirname(__file__), SEARXNG_MSG_FILE)]
+
+
+def extract(
+    # pylint: disable=unused-argument
+    fileobj,
+    keywords,
+    comment_tags,
+    options,
+):
+    """Extract messages from ``searxng.msg`` files by a custom extractor_.
+
+    .. _extractor:
+       https://babel.pocoo.org/en/latest/messages.html#writing-extraction-methods
+    """
+    if fileobj.name not in _MSG_FILES:
+        raise RuntimeError("don't know how to extract messages from %s" % fileobj.name)
+
+    namespace = {}
+    exec(fileobj.read(), {}, namespace)  # pylint: disable=exec-used
+
+    for name in namespace['__all__']:
+        for k, v in namespace[name].items():
+            yield 0, '_', v, ["%s['%s']" % (name, k)]
diff --git a/searx/searxng.msg b/searx/searxng.msg
new file mode 100644
index 000000000..3b876f96d
--- /dev/null
+++ b/searx/searxng.msg
@@ -0,0 +1,52 @@
+# -*- mode: python -*-
+# SPDX-License-Identifier: AGPL-3.0-or-later
+"""A SearXNG message file, see :py:obj:`searx.babel`
+"""
+
+from searx import webutils
+from searx import engines
+
+__all__ = [
+    'CONSTANT_NAMES',
+    'CATEGORY_NAMES',
+    'CATEGORY_GROUPS',
+    'STYLE_NAMES',
+]
+
+CONSTANT_NAMES = {
+    # Constants defined in other modules
+    'DEFAULT_GROUP_NAME': webutils.DEFAULT_GROUP_NAME,
+    'OTHER_CATEGORY': engines.OTHER_CATEGORY,
+}
+
+CATEGORY_NAMES = {
+    'FILES': 'files',
+    'GENERAL': 'general',
+    'MUSIC': 'music',
+    'SOCIAL_MEDIA': 'social media',
+    'IMAGES': 'images',
+    'VIDEOS': 'videos',
+    'IT': 'it',
+    'NEWS': 'news',
+    'MAP': 'map',
+    'ONIONS': 'onions',
+    'SCIENCE': 'science',
+}
+
+CATEGORY_GROUPS = {
+    # non-tab categories
+    'APPS': 'apps',
+    'DICTIONARIES': 'dictionaries',
+    'LYRICS': 'lyrics',
+    'PACKAGES': 'packages',
+    'Q_A': 'q&a',
+    'REPOS': 'repos',
+    'SOFTWARE_WIKIS': 'software wikis',
+    'WEB': 'web',
+}
+
+STYLE_NAMES = {
+    'AUTO': 'auto',
+    'LIGHT': 'light',
+    'DARK': 'dark',
+}
diff --git a/searx/translations/messages.pot b/searx/translations/messages.pot
index 5465c2532..c11451689 100644
--- a/searx/translations/messages.pot
+++ b/searx/translations/messages.pot
@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PROJECT VERSION\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2022-02-21 11:52+0000\n"
+"POT-Creation-Date: 2022-03-16 09:12+0100\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -17,176 +17,208 @@ msgstr ""
 "Content-Transfer-Encoding: 8bit\n"
 "Generated-By: Babel 2.9.1\n"
 
-#: searx/webapp.py:170
+#. CONSTANT_NAMES['DEFAULT_GROUP_NAME']
+#: searx/searxng.msg
+msgid "others"
+msgstr ""
+
+#. CONSTANT_NAMES['OTHER_CATEGORY']
+#: searx/searxng.msg
+msgid "other"
+msgstr ""
+
+#. CATEGORY_NAMES['FILES']
+#: searx/searxng.msg
 msgid "files"
 msgstr ""
 
-#: searx/webapp.py:171
+#. CATEGORY_NAMES['GENERAL']
+#: searx/searxng.msg
 msgid "general"
 msgstr ""
 
-#: searx/webapp.py:172
+#. CATEGORY_NAMES['MUSIC']
+#: searx/searxng.msg
 msgid "music"
 msgstr ""
 
-#: searx/webapp.py:173
+#. CATEGORY_NAMES['SOCIAL_MEDIA']
+#: searx/searxng.msg
 msgid "social media"
 msgstr ""
 
-#: searx/webapp.py:174
+#. CATEGORY_NAMES['IMAGES']
+#: searx/searxng.msg
 msgid "images"
 msgstr ""
 
-#: searx/webapp.py:175
+#. CATEGORY_NAMES['VIDEOS']
+#: searx/searxng.msg
 msgid "videos"
 msgstr ""
 
-#: searx/webapp.py:176
+#. CATEGORY_NAMES['IT']
+#: searx/searxng.msg
 msgid "it"
 msgstr ""
 
-#: searx/webapp.py:177
+#. CATEGORY_NAMES['NEWS']
+#: searx/searxng.msg
 msgid "news"
 msgstr ""
 
-#: searx/webapp.py:178
+#. CATEGORY_NAMES['MAP']
+#: searx/searxng.msg
 msgid "map"
 msgstr ""
 
-#: searx/webapp.py:179
+#. CATEGORY_NAMES['ONIONS']
+#: searx/searxng.msg
 msgid "onions"
 msgstr ""
 
-#: searx/webapp.py:180
+#. CATEGORY_NAMES['SCIENCE']
+#: searx/searxng.msg
 msgid "science"
 msgstr ""
 
-#: searx/webapp.py:182
+#. CATEGORY_GROUPS['APPS']
+#: searx/searxng.msg
 msgid "apps"
 msgstr ""
 
-#: searx/webapp.py:183
+#. CATEGORY_GROUPS['DICTIONARIES']
+#: searx/searxng.msg
 msgid "dictionaries"
 msgstr ""
 
-#: searx/webapp.py:184
+#. CATEGORY_GROUPS['LYRICS']
+#: searx/searxng.msg
 msgid "lyrics"
 msgstr ""
 
-#: searx/webapp.py:185
+#. CATEGORY_GROUPS['PACKAGES']
+#: searx/searxng.msg
 msgid "packages"
 msgstr ""
 
-#: searx/webapp.py:186
+#. CATEGORY_GROUPS['Q_A']
+#: searx/searxng.msg
 msgid "q&a"
 msgstr ""
 
-#: searx/webapp.py:187
+#. CATEGORY_GROUPS['REPOS']
+#: searx/searxng.msg
 msgid "repos"
 msgstr ""
 
-#: searx/webapp.py:188
+#. CATEGORY_GROUPS['SOFTWARE_WIKIS']
+#: searx/searxng.msg
 msgid "software wikis"
 msgstr ""
 
-#: searx/webapp.py:189
+#. CATEGORY_GROUPS['WEB']
+#: searx/searxng.msg
 msgid "web"
 msgstr ""
 
-#: searx/webapp.py:194
+#. STYLE_NAMES['AUTO']
+#: searx/searxng.msg
 msgid "auto"
 msgstr ""
 
-#: searx/webapp.py:194
+#. STYLE_NAMES['LIGHT']
+#: searx/searxng.msg
 msgid "light"
 msgstr ""
 
-#: searx/webapp.py:194
+#. STYLE_NAMES['DARK']
+#: searx/searxng.msg
 msgid "dark"
 msgstr ""
 
-#: searx/webapp.py:197
+#: searx/webapp.py:168
 msgid "timeout"
 msgstr ""
 
-#: searx/webapp.py:198
+#: searx/webapp.py:169
 msgid "parsing error"
 msgstr ""
 
-#: searx/webapp.py:199
+#: searx/webapp.py:170
 msgid "HTTP protocol error"
 msgstr ""
 
-#: searx/webapp.py:200
+#: searx/webapp.py:171
 msgid "network error"
 msgstr ""
 
-#: searx/webapp.py:202
+#: searx/webapp.py:173
 msgid "unexpected crash"
 msgstr ""
 
-#: searx/webapp.py:209
+#: searx/webapp.py:180
 msgid "HTTP error"
 msgstr ""
 
-#: searx/webapp.py:210
+#: searx/webapp.py:181
 msgid "HTTP connection error"
 msgstr ""
 
-#: searx/webapp.py:216
+#: searx/webapp.py:187
 msgid "proxy error"
 msgstr ""
 
-#: searx/webapp.py:217
+#: searx/webapp.py:188
 msgid "CAPTCHA"
 msgstr ""
 
-#: searx/webapp.py:218
+#: searx/webapp.py:189
 msgid "too many requests"
 msgstr ""
 
-#: searx/webapp.py:219
+#: searx/webapp.py:190
 msgid "access denied"
 msgstr ""
 
-#: searx/webapp.py:220
+#: searx/webapp.py:191
 msgid "server API error"
 msgstr ""
 
-#: searx/webapp.py:431
+#: searx/webapp.py:409
 msgid "No item found"
 msgstr ""
 
 #: searx/engines/qwant.py:212
-#: searx/templates/simple/result_templates/images.html:23 searx/webapp.py:433
+#: searx/templates/simple/result_templates/images.html:23 searx/webapp.py:411
 msgid "Source"
 msgstr ""
 
-#: searx/webapp.py:435
+#: searx/webapp.py:413
 msgid "Error loading the next page"
 msgstr ""
 
-#: searx/webapp.py:547 searx/webapp.py:967
+#: searx/webapp.py:525 searx/webapp.py:979
 msgid "Invalid settings, please edit your preferences"
 msgstr ""
 
-#: searx/webapp.py:563
+#: searx/webapp.py:541
 msgid "Invalid settings"
 msgstr ""
 
-#: searx/webapp.py:639 searx/webapp.py:714
+#: searx/webapp.py:617 searx/webapp.py:693
 msgid "search error"
 msgstr ""
 
-#: searx/webapp.py:757
+#: searx/webapp.py:739
 msgid "{minutes} minute(s) ago"
 msgstr ""
 
-#: searx/webapp.py:759
+#: searx/webapp.py:741
 msgid "{hours} hour(s), {minutes} minute(s) ago"
 msgstr ""
 
-#: searx/webapp.py:875
+#: searx/webapp.py:870
 msgid "Suspended"
 msgstr ""
 
@@ -390,7 +422,7 @@ msgstr ""
 #: searx/templates/oscar/preferences.html:362
 #: searx/templates/oscar/preferences.html:425
 #: searx/templates/simple/preferences.html:295
-#: searx/templates/simple/preferences.html:344
+#: searx/templates/simple/preferences.html:346
 msgid "Allow"
 msgstr ""
 
@@ -496,13 +528,13 @@ msgid "Engines"
 msgstr ""
 
 #: searx/templates/oscar/preferences.html:105
-#: searx/templates/simple/preferences.html:339
+#: searx/templates/simple/preferences.html:341
 msgid "Special Queries"
 msgstr ""
 
 #: searx/templates/oscar/preferences.html:106
 #: searx/templates/oscar/preferences.html:465
-#: searx/templates/simple/preferences.html:379
+#: searx/templates/simple/preferences.html:381
 msgid "Cookies"
 msgstr ""
 
@@ -779,78 +811,78 @@ msgid "Query"
 msgstr ""
 
 #: searx/templates/oscar/preferences.html:426
-#: searx/templates/simple/preferences.html:345
+#: searx/templates/simple/preferences.html:347
 msgid "Keywords"
 msgstr ""
 
 #: searx/templates/oscar/preferences.html:427
-#: searx/templates/simple/preferences.html:346
+#: searx/templates/simple/preferences.html:348
 msgid "Name"
 msgstr ""
 
 #: searx/templates/oscar/preferences.html:428
-#: searx/templates/simple/preferences.html:347
+#: searx/templates/simple/preferences.html:349
 msgid "Description"
 msgstr ""
 
 #: searx/templates/oscar/preferences.html:429
-#: searx/templates/simple/preferences.html:348
+#: searx/templates/simple/preferences.html:350
 msgid "Examples"
 msgstr ""
 
 #: searx/templates/oscar/preferences.html:434
-#: searx/templates/simple/preferences.html:351
+#: searx/templates/simple/preferences.html:353
 msgid "This is the list of SearXNG's instant answering modules."
 msgstr ""
 
 #: searx/templates/oscar/preferences.html:447
-#: searx/templates/simple/preferences.html:362
+#: searx/templates/simple/preferences.html:364
 msgid "This is the list of plugins."
 msgstr ""
 
 #: searx/templates/oscar/preferences.html:468
-#: searx/templates/simple/preferences.html:381
+#: searx/templates/simple/preferences.html:383
 msgid ""
 "This is the list of cookies and their values SearXNG is storing on your "
 "computer."
 msgstr ""
 
 #: searx/templates/oscar/preferences.html:469
-#: searx/templates/simple/preferences.html:382
+#: searx/templates/simple/preferences.html:384
 msgid "With that list, you can assess SearXNG transparency."
 msgstr ""
 
 #: searx/templates/oscar/preferences.html:474
-#: searx/templates/simple/preferences.html:387
+#: searx/templates/simple/preferences.html:389
 msgid "Cookie name"
 msgstr ""
 
 #: searx/templates/oscar/preferences.html:475
-#: searx/templates/simple/preferences.html:388
+#: searx/templates/simple/preferences.html:390
 msgid "Value"
 msgstr ""
 
 #: searx/templates/oscar/preferences.html:492
-#: searx/templates/simple/preferences.html:409
+#: searx/templates/simple/preferences.html:411
 msgid ""
 "These settings are stored in your cookies, this allows us not to store "
 "this data about you."
 msgstr ""
 
 #: searx/templates/oscar/preferences.html:493
-#: searx/templates/simple/preferences.html:411
+#: searx/templates/simple/preferences.html:413
 msgid ""
 "These cookies serve your sole convenience, we don't use these cookies to "
 "track you."
 msgstr ""
 
 #: searx/templates/oscar/preferences.html:497
-#: searx/templates/simple/preferences.html:400
+#: searx/templates/simple/preferences.html:402
 msgid "Search URL of the currently saved preferences"
 msgstr ""
 
 #: searx/templates/oscar/preferences.html:498
-#: searx/templates/simple/preferences.html:404
+#: searx/templates/simple/preferences.html:406
 msgid ""
 "Note: specifying custom settings in the search URL can reduce privacy by "
 "leaking data to the clicked result sites."
@@ -865,7 +897,7 @@ msgid "back"
 msgstr ""
 
 #: searx/templates/oscar/preferences.html:505
-#: searx/templates/simple/preferences.html:415
+#: searx/templates/simple/preferences.html:417
 msgid "Reset defaults"
 msgstr ""
 
@@ -1210,7 +1242,7 @@ msgstr ""
 msgid "Length"
 msgstr ""
 
-#: searx/templates/simple/categories.html:16
+#: searx/templates/simple/categories.html:24
 msgid "Click on the magnifier to perform search"
 msgstr ""
 
@@ -1238,11 +1270,11 @@ msgstr ""
 msgid "Supports selected language"
 msgstr ""
 
-#: searx/templates/simple/preferences.html:414
+#: searx/templates/simple/preferences.html:416
 msgid "Save"
 msgstr ""
 
-#: searx/templates/simple/preferences.html:416
+#: searx/templates/simple/preferences.html:418
 msgid "Back"
 msgstr ""
 
@@ -1250,11 +1282,11 @@ msgstr ""
 msgid "Answers"
 msgstr ""
 
-#: searx/templates/simple/results.html:164
+#: searx/templates/simple/results.html:162
 msgid "Previous page"
 msgstr ""
 
-#: searx/templates/simple/results.html:181
+#: searx/templates/simple/results.html:179
 msgid "Next page"
 msgstr ""
 
diff --git a/searx/webapp.py b/searx/webapp.py
index 8c3e78deb..5aa0a382c 100755
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -80,7 +80,6 @@ from searx.webutils import (
     new_hmac,
     is_hmac_of,
     is_flask_run_cmdline,
-    DEFAULT_GROUP_NAME,
     group_engines_in_tab,
 )
 from searx.webadapter import (
@@ -167,35 +166,6 @@ app.secret_key = settings['server']['secret_key']
 
 babel = Babel(app)
 
-# used when translating category names
-_category_names = (
-    gettext('files'),
-    gettext('general'),
-    gettext('music'),
-    gettext('social media'),
-    gettext('images'),
-    gettext('videos'),
-    gettext('it'),
-    gettext('news'),
-    gettext('map'),
-    gettext('onions'),
-    gettext('science'),
-    # non-tab categories
-    gettext('apps'),
-    gettext('dictionaries'),
-    gettext('lyrics'),
-    gettext('packages'),
-    gettext('q&a'),
-    gettext('repos'),
-    gettext('software wikis'),
-    gettext('web'),
-    gettext(DEFAULT_GROUP_NAME),
-    gettext(OTHER_CATEGORY),
-)
-
-_simple_style = (gettext('auto'), gettext('light'), gettext('dark'))
-
-#
 timeout_text = gettext('timeout')
 parsing_error_text = gettext('parsing error')
 http_protocol_error_text = gettext('HTTP protocol error')