refactor engine's search language handling

Add match_language function in utils to match any user given
language code with a list of engine's supported languages.

Also add language_aliases dict on each engine to translate
standard language codes into the custom codes used by the engine.

1 files changed, 3 insertions, 15 deletions

diff --git a/utils/fetch_languages.py b/utils/fetch_languages.py
index 4aabf68bf..dd7aa659f 100644
--- a/utils/fetch_languages.py
+++ b/utils/fetch_languages.py
@@ -19,19 +19,6 @@ from searx.engines import initialize_engines, engines
 engines_languages_file = 'engines_languages.json'
 languages_file = 'languages.py'
 
-# custom fixes for non standard locale codes
-# sl-SL is technically not invalid, but still a mistake
-# TODO: move to respective engines
-locale_fixes = {
-    'sl-sl': 'sl-SI',
-    'ar-xa': 'ar-SA',
-    'es-xl': 'es-419',
-    'zh-chs': 'zh-Hans-CN',
-    'zh-cht': 'zh-Hant-TW',
-    'tzh-tw': 'zh-Hant-TW',
-    'tzh-hk': 'zh-Hant-HK'
-}
-
 
 # Fetchs supported languages for each engine and writes json file with those.
 def fetch_supported_languages():
@@ -76,8 +63,9 @@ def join_language_lists(engines_languages):
         for lang_code in engines_languages[engine_name]:
 
             # apply custom fixes if necessary
-            if lang_code.lower() in locale_fixes:
-                lang_code = locale_fixes[lang_code.lower()]
+            if lang_code in getattr(engines[engine_name], 'language_aliases', {}).values():
+                lang_code = next(lc for lc, alias in engines[engine_name].language_aliases.items()
+                                 if lang_code == alias)
 
             locale = get_locale(lang_code)