Merge pull request #247 from dalf/remote_settings_locales

settings.yml: remove locales section.

9 files changed, 97 insertions, 64 deletions

diff --git a/docs/src/searx.locales.rst b/docs/src/searx.locales.rst
new file mode 100644
index 000000000..579247aff
--- /dev/null
+++ b/docs/src/searx.locales.rst
@@ -0,0 +1,8 @@
+.. _searx.locales:
+
+=======
+Locales
+=======
+
+.. automodule:: searx.locales
+  :members:
diff --git a/searx/locales.py b/searx/locales.py
new file mode 100644
index 000000000..80defceb7
--- /dev/null
+++ b/searx/locales.py
@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+# SPDX-License-Identifier: AGPL-3.0-or-later
+# lint: pylint
+"""Initialize :py:obj:`LOCALE_NAMES`, :py:obj:`UI_LOCALE_CODES` and
+:py:obj:`RTL_LOCALES`."""
+
+from typing import List, Set
+import os
+import pathlib
+
+from babel import Locale
+
+LOCALE_NAMES = {
+    "oc": "Occitan",
+    "nl_BE": "Vlaams (Dutch, Belgium)",
+}
+"""Mapping of locales and their description.  Locales e.g. 'fr' or 'pt_BR'
+(delimiter is *underline* '_')"""
+
+UI_LOCALE_CODES: List[str] = []
+"""List of locales e.g. 'fr' or 'pt-BR' (delimiter is '-')"""
+
+RTL_LOCALES: Set[str] = set()
+"""List of *Right-To-Left* locales e.g. 'he' or 'fa_IR' (delimiter is
+*underline* '_')"""
+
+
+def _get_name(locale, language_code):
+    language_name = locale.get_language_name(language_code).capitalize()
+    if language_name and ('a' <= language_name[0] <= 'z'):
+        language_name = language_name.capitalize()
+    terrirtory_name = locale.get_territory_name(language_code)
+    return language_name, terrirtory_name
+
+
+def _get_locale_name(locale, locale_name):
+    """Get locale name e.g. 'Français - fr' or 'Português (Brasil) - pt-BR'
+
+    :param locale: instance of :py:class:`Locale`
+    :param locale_name: name e.g. 'fr'  or 'pt_BR'
+    """
+    native_language, native_territory = _get_name(locale, locale_name)
+    english_language, english_territory = _get_name(locale, 'en')
+    if native_territory == english_territory:
+        english_territory = None
+    if not native_territory and not english_territory:
+        if native_language == english_language:
+            return native_language
+        return native_language + ' (' + english_language + ')'
+    result = native_language + ', ' + native_territory + ' (' + english_language
+    if english_territory:
+        return result + ', ' + english_territory + ')'
+    return result + ')'
+
+
+def initialize_locales(directory):
+    """Initialize global names :py:obj:`LOCALE_NAMES`, :py:obj:`UI_LOCALE_CODES` and
+    :py:obj:`RTL_LOCALES`.
+    """
+    global LOCALE_NAMES, UI_LOCALE_CODES, RTL_LOCALES  # pylint: disable=global-statement
+    for dirname in sorted(os.listdir(directory)):
+        # Based on https://flask-babel.tkte.ch/_modules/flask_babel.html#Babel.list_translations
+        if not os.path.isdir( os.path.join(directory, dirname, 'LC_MESSAGES') ):
+            continue
+        info = LOCALE_NAMES.get(dirname)
+        if not info:
+            locale = Locale.parse(dirname)
+            LOCALE_NAMES[dirname] = _get_locale_name(locale, dirname)
+            if locale.text_direction == 'rtl':
+                RTL_LOCALES.add(dirname)
+
+    UI_LOCALE_CODES = [l.replace('_', '-') for l in LOCALE_NAMES]
+
+
+initialize_locales(pathlib.Path(__file__).parent / 'translations')
diff --git a/searx/preferences.py b/searx/preferences.py
index e4a2ebecd..69832c052 100644
--- a/searx/preferences.py
+++ b/searx/preferences.py
@@ -11,6 +11,7 @@ from urllib.parse import parse_qs, urlencode
 
 from searx import settings, autocomplete
 from searx.languages import language_codes as languages
+from searx.locales import LOCALE_NAMES
 from searx.webutils import VALID_LANGUAGE_CODE
 
 
@@ -340,7 +341,7 @@ class Preferences:
             'locale': EnumStringSetting(
                 settings['ui']['default_locale'],
                 is_locked('locale'),
-                choices=list(settings['locales'].keys()) + ['']
+                choices=list(LOCALE_NAMES.keys()) + ['']
             ),
             'autocomplete': EnumStringSetting(
                 settings['search']['autocomplete'],
diff --git a/searx/settings.yml b/searx/settings.yml
index 009ea6856..f4321b89d 100644
--- a/searx/settings.yml
+++ b/searx/settings.yml
@@ -1651,53 +1651,6 @@ engines:
 #        chars: ' '
 #        keys: ['line']
 
-locales:
-  en: English
-  ar: العَرَبِيَّة (Arabic)
-  bg: Български (Bulgarian)
-  bo: བོད་སྐད་ (Tibetian)
-  ca: Català (Catalan)
-  cs: Čeština (Czech)
-  cy: Cymraeg (Welsh)
-  da: Dansk (Danish)
-  de: Deutsch (German)
-  el_GR: Ελληνικά (Greek_Greece)
-  eo: Esperanto (Esperanto)
-  es: Español (Spanish)
-  et: Eesti (Estonian)
-  eu: Euskara (Basque)
-  fa_IR: (fārsī) فارسى (Persian)
-  fi: Suomi (Finnish)
-  fil: Wikang Filipino (Filipino)
-  fr: Français (French)
-  gl: Galego (Galician)
-  he: עברית (Hebrew)
-  hr: Hrvatski (Croatian)
-  hu: Magyar (Hungarian)
-  ia: Interlingua (Interlingua)
-  it: Italiano (Italian)
-  ja: 日本語 (Japanese)
-  lt: Lietuvių (Lithuanian)
-  nl: Nederlands (Dutch)
-  nl_BE: Vlaams (Dutch_Belgium)
-  oc: Lenga D'òc (Occitan)
-  pl: Polski (Polish)
-  pt: Português (Portuguese)
-  pt_BR: Português (Portuguese_Brazil)
-  ro: Română (Romanian)
-  ru: Русский (Russian)
-  sk: Slovenčina (Slovak)
-  sl: Slovenski (Slovene)
-  sr: српски (Serbian)
-  sv: Svenska (Swedish)
-  te: తెలుగు (telugu)
-  ta: தமிழ் (Tamil)
-  tr: Türkçe (Turkish)
-  uk: українська мова (Ukrainian)
-  vi: tiếng việt (Vietnamese)
-  zh: 中文 (Chinese)
-  zh_TW: 國語 (Taiwanese Mandarin)
-
 doi_resolvers:
   oadoi.org: 'https://oadoi.org/'
   doi.org: 'https://doi.org/'
diff --git a/searx/testing.py b/searx/testing.py
index e5c1bd313..b31ba8997 100644
--- a/searx/testing.py
+++ b/searx/testing.py
@@ -79,7 +79,7 @@ class SearxRobotLayer():
 def run_robot_tests(tests):
     print('Running {0} tests'.format(len(tests)))
     for test in tests:
-        with Browser('firefox', headless=True) as browser:
+        with Browser('firefox', headless=True, profile_preferences={'intl.accept_languages': 'en'}) as browser:
             test(browser)
 
 
diff --git a/searx/webapp.py b/searx/webapp.py
index d638dfb55..0bf1a37f5 100755
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -106,6 +106,7 @@ from searx.flaskfix import patch_application
 
 from searx.autocomplete import search_autocomplete, backends as autocomplete_backends
 from searx.languages import language_codes as languages
+from searx.locales import LOCALE_NAMES, UI_LOCALE_CODES, RTL_LOCALES
 from searx.search import SearchWithPlugins, initialize as search_initialize
 from searx.network import stream as http_stream
 from searx.search.checker import get_result as checker_get_result
@@ -176,12 +177,6 @@ if (not werkzeug_reloader
 
 babel = Babel(app)
 
-rtl_locales = [
-    'ar', 'arc', 'bcc', 'bqi', 'ckb', 'dv', 'fa', 'fa_IR', 'glk', 'he',
-    'ku', 'mzn', 'pnb', 'ps', 'sd', 'ug', 'ur', 'yi'
-]
-ui_locale_codes = [l.replace('_', '-') for l in settings['locales'].keys()]
-
 # used when translating category names
 _category_names = (
     gettext('files'),
@@ -258,7 +253,7 @@ def _get_browser_or_settings_language(req, lang_list):
 @babel.localeselector
 def get_locale():
     if 'locale' in request.form\
-       and request.form['locale'] in settings['locales']:
+       and request.form['locale'] in LOCALE_NAMES:
         # use locale from the form
         locale = request.form['locale']
         locale_source = 'form'
@@ -268,7 +263,7 @@ def get_locale():
         locale_source = 'preferences'
     else:
         # use local from the browser
-        locale = _get_browser_or_settings_language(request, ui_locale_codes)
+        locale = _get_browser_or_settings_language(request, UI_LOCALE_CODES)
         locale = locale.replace('-', '_')
         locale_source = 'browser'
 
@@ -463,7 +458,7 @@ def render(template_name, override_theme=None, **kwargs):
     kwargs['translations'] = json.dumps(get_translations(), separators=(',', ':'))
 
     locale = request.preferences.get_value('locale')
-    if locale in rtl_locales and 'rtl' not in kwargs:
+    if locale in RTL_LOCALES and 'rtl' not in kwargs:
         kwargs['rtl'] = True
     if 'current_language' not in kwargs:
         kwargs['current_language'] = match_language(
@@ -1042,7 +1037,7 @@ def preferences():
     return render(
         'preferences.html',
         selected_categories = get_selected_categories(request.preferences, request.form),
-        locales = settings['locales'],
+        locales = LOCALE_NAMES,
         current_locale = request.preferences.get_value("locale"),
         image_proxy = image_proxy,
         engines_by_category = engines_by_category,
@@ -1315,7 +1310,7 @@ def config():
         'engines': _engines,
         'plugins': _plugins,
         'instance_name': settings['general']['instance_name'],
-        'locales': settings['locales'],
+        'locales': LOCALE_NAMES,
         'default_locale': settings['ui']['default_locale'],
         'autocomplete': settings['search']['autocomplete'],
         'safe_search': settings['search']['safe_search'],
diff --git a/searx_extra/update/update_currencies.py b/searx_extra/update/update_currencies.py
index 0cfb7a951..a572f4e9d 100755
--- a/searx_extra/update/update_currencies.py
+++ b/searx_extra/update/update_currencies.py
@@ -8,7 +8,8 @@ import json
 from sys import path
 from os.path import realpath, dirname, join
 
-from searx import searx_dir, settings
+from searx import searx_dir
+from searx.locales import LOCALE_NAMES
 from searx.engines.wikidata import send_wikidata_query
 
 
@@ -44,7 +45,7 @@ ORDER BY ?iso4217 ?article_name
 """
 
 
-LANGUAGES = settings['locales'].keys()
+LANGUAGES = LOCALE_NAMES.keys()
 LANGUAGES_SPARQL = ', '.join(set(map(lambda l: repr(l.split('_')[0]), LANGUAGES)))
 
 
diff --git a/searx_extra/update/update_engine_descriptions.py b/searx_extra/update/update_engine_descriptions.py
index cf9007da3..37be77177 100755
--- a/searx_extra/update/update_engine_descriptions.py
+++ b/searx_extra/update/update_engine_descriptions.py
@@ -8,6 +8,7 @@ from lxml.html import fromstring
 
 from searx.engines.wikidata import send_wikidata_query
 from searx.utils import extract_text
+from searx.locales import LOCALE_NAMES
 import searx
 import searx.search
 import searx.network
@@ -35,7 +36,7 @@ WHERE {
 ORDER BY ?itemLang
 """
 
-LANGUAGES = searx.settings['locales'].keys()
+LANGUAGES = LOCALE_NAMES.keys()
 LANGUAGES_SPARQL = ', '.join(set(map(lambda l: repr(l.split('_')[0]), LANGUAGES)))
 IDS = None
 
diff --git a/tests/unit/test_settings_loader.py b/tests/unit/test_settings_loader.py
index 7df64e524..a3257ee4b 100644
--- a/tests/unit/test_settings_loader.py
+++ b/tests/unit/test_settings_loader.py
@@ -41,7 +41,6 @@ class TestDefaultSettings(SearxTestCase):
         self.assertTrue(isinstance(settings['server']['port'], int))
         self.assertTrue(isinstance(settings['server']['bind_address'], str))
         self.assertTrue(isinstance(settings['engines'], list))
-        self.assertTrue(isinstance(settings['locales'], dict))
         self.assertTrue(isinstance(settings['doi_resolvers'], dict))
         self.assertTrue(isinstance(settings['default_doi_resolver'], str))