diff options
Diffstat (limited to 'searx/engines/startpage.py')
| -rw-r--r-- | searx/engines/startpage.py | 183 |
1 files changed, 176 insertions, 7 deletions
diff --git a/searx/engines/startpage.py b/searx/engines/startpage.py index f857f7b6d..75435c3fa 100644 --- a/searx/engines/startpage.py +++ b/searx/engines/startpage.py @@ -1,6 +1,63 @@ # SPDX-License-Identifier: AGPL-3.0-or-later # lint: pylint -"""Startpage (Web) +"""Startpage's language & region selectors are a mess .. + +.. _startpage regions: + +Startpage regions +================= + +In the list of regions there are tags we need to map to common region tags:: + + pt-BR_BR --> pt_BR + zh-CN_CN --> zh_Hans_CN + zh-TW_TW --> zh_Hant_TW + zh-TW_HK --> zh_Hant_HK + en-GB_GB --> en_GB + +and there is at least one tag with a three letter language tag (ISO 639-2):: + + fil_PH --> fil_PH + +The locale code ``no_NO`` from Startpage does not exists and is mapped to +``nb-NO``:: + + babel.core.UnknownLocaleError: unknown locale 'no_NO' + +For reference see languages-subtag at iana; ``no`` is the macrolanguage [1]_ and +W3C recommends subtag over macrolanguage [2]_. + +.. [1] `iana: language-subtag-registry + <https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry>`_ :: + + type: language + Subtag: nb + Description: Norwegian Bokmål + Added: 2005-10-16 + Suppress-Script: Latn + Macrolanguage: no + +.. [2] + Use macrolanguages with care. Some language subtags have a Scope field set to + macrolanguage, i.e. this primary language subtag encompasses a number of more + specific primary language subtags in the registry. ... As we recommended for + the collection subtags mentioned above, in most cases you should try to use + the more specific subtags ... `W3: The primary language subtag + <https://www.w3.org/International/questions/qa-choosing-language-tags#langsubtag>`_ + +.. _startpage languages: + +Startpage languages +=================== + +The displayed name in Startpage's settings page depend on the location of the IP +when the 'Accept-Language' HTTP header is unset (in the language update script +we use "en-US,en;q=0.5" to get uniform names independent from the IP). + +Each option has a displayed name and a value, either of which may represent the +language name in the native script, the language name in English, an English +transliteration of the native name, the English name of the writing script used +by the language, or occasionally something else entirely. """ @@ -16,13 +73,16 @@ from lxml import html from babel import Locale from babel.localedata import locale_identifiers -from searx.network import get +from searx import network from searx.utils import extract_text, eval_xpath, match_language from searx.exceptions import ( SearxEngineResponseException, SearxEngineCaptchaException, ) +from searx.enginelib.traits import EngineTraits + +traits: EngineTraits # about about = { @@ -66,11 +126,11 @@ def raise_captcha(resp): def get_sc_code(headers): - """Get an actual `sc` argument from startpage's home page. + """Get an actual ``sc`` argument from Startpage's home page. - Startpage puts a `sc` argument on every link. Without this argument - startpage considers the request is from a bot. We do not know what is - encoded in the value of the `sc` argument, but it seems to be a kind of a + Startpage puts a ``sc`` argument on every link. Without this argument + Startpage considers the request is from a bot. We do not know what is + encoded in the value of the ``sc`` argument, but it seems to be a kind of a *time-stamp*. This *time-stamp* is valid for a few hours. This function scrap a new *time-stamp* from startpage's home page every hour @@ -83,7 +143,7 @@ def get_sc_code(headers): if time() > (sc_code_ts + 3000): logger.debug("query new sc time-stamp ...") - resp = get(base_url, headers=headers) + resp = network.get(base_url, headers=headers) raise_captcha(resp) dom = html.fromstring(resp.text) @@ -258,3 +318,112 @@ def _fetch_supported_languages(resp): print('Unknown language option in Startpage: {} ({})'.format(sp_option_value, sp_option_text)) return supported_languages + + +def fetch_traits(engine_traits: EngineTraits): + """Fetch :ref:`languages <startpage languages>` and :ref:`regions <startpage + regions>` from Startpage.""" + # pylint: disable=import-outside-toplevel, too-many-locals, too-many-branches + # pylint: disable=too-many-statements + + engine_traits.data_type = 'supported_languages' # deprecated + + import babel + from searx.utils import gen_useragent + from searx.locales import region_tag + + headers = { + 'User-Agent': gen_useragent(), + 'Accept-Language': "en-US,en;q=0.5", # bing needs to set the English language + } + resp = network.get('https://www.startpage.com/do/settings', headers=headers) + + if not resp.ok: + print("ERROR: response from Startpage is not OK.") + + dom = html.fromstring(resp.text) + + # regions + + sp_region_names = [] + for option in dom.xpath('//form[@name="settings"]//select[@name="search_results_region"]/option'): + sp_region_names.append(option.get('value')) + + for eng_tag in sp_region_names: + if eng_tag == 'all': + continue + babel_region_tag = {'no_NO': 'nb_NO'}.get(eng_tag, eng_tag) # norway + + if '-' in babel_region_tag: + l, r = babel_region_tag.split('-') + r = r.split('_')[-1] + sxng_tag = region_tag(babel.Locale.parse(l + '_' + r, sep='_')) + + else: + try: + sxng_tag = region_tag(babel.Locale.parse(babel_region_tag, sep='_')) + + except babel.UnknownLocaleError: + print("ERROR: can't determine babel locale of startpage's locale %s" % eng_tag) + continue + + conflict = engine_traits.regions.get(sxng_tag) + if conflict: + if conflict != eng_tag: + print("CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, eng_tag)) + continue + engine_traits.regions[sxng_tag] = eng_tag + + # languages + + catalog_engine2code = {name.lower(): lang_code for lang_code, name in babel.Locale('en').languages.items()} + + # get the native name of every language known by babel + + for lang_code in filter(lambda lang_code: lang_code.find('_') == -1, babel.localedata.locale_identifiers()): + native_name = babel.Locale(lang_code).get_language_name().lower() + # add native name exactly as it is + catalog_engine2code[native_name] = lang_code + + # add "normalized" language name (i.e. français becomes francais and español becomes espanol) + unaccented_name = ''.join(filter(lambda c: not combining(c), normalize('NFKD', native_name))) + if len(unaccented_name) == len(unaccented_name.encode()): + # add only if result is ascii (otherwise "normalization" didn't work) + catalog_engine2code[unaccented_name] = lang_code + + # values that can't be determined by babel's languages names + + catalog_engine2code.update( + { + # traditional chinese used in .. + 'fantizhengwen': 'zh_Hant', + # Korean alphabet + 'hangul': 'ko', + # Malayalam is one of 22 scheduled languages of India. + 'malayam': 'ml', + 'norsk': 'nb', + 'sinhalese': 'si', + } + ) + + skip_eng_tags = { + 'english_uk', # SearXNG lang 'en' already maps to 'english' + } + + for option in dom.xpath('//form[@name="settings"]//select[@name="language"]/option'): + + eng_tag = option.get('value') + if eng_tag in skip_eng_tags: + continue + name = extract_text(option).lower() + + sxng_tag = catalog_engine2code.get(eng_tag) + if sxng_tag is None: + sxng_tag = catalog_engine2code[name] + + conflict = engine_traits.languages.get(sxng_tag) + if conflict: + if conflict != eng_tag: + print("CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, eng_tag)) + continue + engine_traits.languages[sxng_tag] = eng_tag |