[mod] DuckDuckGo: reversed engineered & upgrade to data_type: traits_v1

Partial reverse engineering of the DuckDuckGo (DDG) engines including a
improved language and region handling based on the enigne.traits_v1 data.

- DDG Lite
- DDG Instant Answer API
- DDG Images
- DDG Weather

docs/src/searx.engine.duckduckgo.rst:
  Online documentation of the DDG engines (make docs.live)

searx/data/engine_traits.json
  Add data type "traits_v1" generated by the fetch_traits() functions from:

  - "duckduckgo" (WEB),
  - "duckduckgo images" and
  - "duckduckgo weather"

  and remove data from obsolete data type "supported_languages".

searx/autocomplete.py:
  Reversed engineered Autocomplete from DDG.  Supports DDG's languages.

searx/engines/duckduckgo.py:
  - fetch_traits():  Fetch languages & regions from DDG.

  - get_ddg_lang(): Get DDG's language identifier from SearXNG's locale.  DDG
    defines its languages by region codes.  DDG-Lite does not offer a language
    selection to the user, only a region can be selected by the user.

  - Cache ``vqd`` value: The vqd value depends on the query string and is needed
    for the follow up pages or the images loaded by a XMLHttpRequest (DDG
    images).  The ``vqd`` value of a search term is stored for 10min in the
    redis DB.

  - DDG Lite engine: reversed engineered request method with improved Language
    and region support and better ``vqd`` handling.

searx/engines/duckduckgo_definitions.py: DDG Instant Answer API
  The *instant answers* API does not support languages, or at least we could not
  find out how language support should work.  It seems that most of the features
  are based on English terms.

searx/engines/duckduckgo_images.py: DDG Images
  Reversed engineered request method.  Improved language and region handling
  based on cookies and the enigne.traits_v1 data.  Response: add image format to
  the result list

searx/engines/duckduckgo_weather.py: DDG Weather
  Improved language and region handling based on cookies and the
  enigne.traits_v1 data.

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

1 files changed, 257 insertions, 86 deletions

diff --git a/searx/engines/duckduckgo.py b/searx/engines/duckduckgo.py
index cb47122ae..85e977bdb 100644
--- a/searx/engines/duckduckgo.py
+++ b/searx/engines/duckduckgo.py
@@ -1,73 +1,207 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 # lint: pylint
-"""DuckDuckGo Lite
+"""
+DuckDuckGo Lite
+~~~~~~~~~~~~~~~
 """
 
+from typing import TYPE_CHECKING
+from urllib.parse import urlencode
 import json
-from lxml import html
+import babel
+import lxml.html
 
+from searx import (
+    network,
+    locales,
+    redislib,
+)
+from searx import redisdb
 from searx.utils import (
-    dict_subset,
     eval_xpath,
     eval_xpath_getindex,
     extract_text,
-    match_language,
 )
-from searx import network
 from searx.enginelib.traits import EngineTraits
+from searx.exceptions import SearxEngineAPIException
+
+if TYPE_CHECKING:
+    import logging
+
+    logger: logging.Logger
 
 traits: EngineTraits
 
-# about
 about = {
     "website": 'https://lite.duckduckgo.com/lite/',
     "wikidata_id": 'Q12805',
-    "official_api_documentation": 'https://duckduckgo.com/api',
     "use_official_api": False,
     "require_api_key": False,
     "results": 'HTML',
 }
 
+send_accept_language_header = True
+"""DuckDuckGo-Lite tries to guess user's prefered language from the HTTP
+``Accept-Language``.  Optional the user can select a region filter (but not a
+language).
+"""
+
 # engine dependent config
 categories = ['general', 'web']
 paging = True
-supported_languages_url = 'https://duckduckgo.com/util/u588.js'
 time_range_support = True
-send_accept_language_header = True
+safesearch = True  # user can't select but the results are filtered
 
-language_aliases = {
-    'ar-SA': 'ar-XA',
-    'es-419': 'es-XL',
-    'ja': 'jp-JP',
-    'ko': 'kr-KR',
-    'sl-SI': 'sl-SL',
-    'zh-TW': 'tzh-TW',
-    'zh-HK': 'tzh-HK',
-}
+url = 'https://lite.duckduckgo.com/lite/'
+# url_ping = 'https://duckduckgo.com/t/sl_l'
 
 time_range_dict = {'day': 'd', 'week': 'w', 'month': 'm', 'year': 'y'}
+form_data = {'v': 'l', 'api': 'd.js', 'o': 'json'}
 
-# search-url
-url = 'https://lite.duckduckgo.com/lite/'
-url_ping = 'https://duckduckgo.com/t/sl_l'
 
-# match query's language to a region code that duckduckgo will accept
-def get_region_code(lang, lang_list=None):
-    if lang == 'all':
-        return None
+def cache_vqd(query, value):
+    """Caches a ``vqd`` value from a query.
+
+    The vqd value depends on the query string and is needed for the follow up
+    pages or the images loaded by a XMLHttpRequest:
+
+    - DuckDuckGo Web: `https://links.duckduckgo.com/d.js?q=...&vqd=...`
+    - DuckDuckGo Images: `https://duckduckgo.com/i.js??q=...&vqd=...`
+
+    """
+    c = redisdb.client()
+    if c:
+        logger.debug("cache vqd value: %s", value)
+        key = 'SearXNG_ddg_vqd' + redislib.secret_hash(query)
+        c.set(key, value, ex=600)
+
+
+def get_vqd(query, headers):
+    """Returns the ``vqd`` that fits to the *query*.  If there is no ``vqd`` cached
+    (:py:obj:`cache_vqd`) the query is sent to DDG to get a vqd value from the
+    response.
+
+    """
+    value = None
+    c = redisdb.client()
+    if c:
+        key = 'SearXNG_ddg_vqd' + redislib.secret_hash(query)
+        value = c.get(key)
+        if value:
+            value = value.decode('utf-8')
+            logger.debug("re-use cached vqd value: %s", value)
+            return value
 
-    lang_code = match_language(lang, lang_list or [], language_aliases, 'wt-WT')
-    lang_parts = lang_code.split('-')
+    query_url = 'https://duckduckgo.com/?{query}&iar=images'.format(query=urlencode({'q': query}))
+    res = network.get(query_url, headers=headers)
+    content = res.text
+    if content.find('vqd=\'') == -1:
+        raise SearxEngineAPIException('Request failed')
+    value = content[content.find('vqd=\'') + 5 :]
+    value = value[: value.find('\'')]
+    logger.debug("new vqd value: %s", value)
+    cache_vqd(query, value)
+    return value
 
-    # country code goes first
-    return lang_parts[1].lower() + '-' + lang_parts[0].lower()
+
+def get_ddg_lang(eng_traits: EngineTraits, sxng_locale, default='en_US'):
+    """Get DuckDuckGo's language identifier from SearXNG's locale.
+
+    DuckDuckGo defines its lanaguages by region codes (see
+    :py:obj:`fetch_traits`).
+
+    To get region and language of a DDG service use:
+
+    .. code: python
+
+       eng_region = traits.get_region(params['searxng_locale'], traits.all_locale)
+       eng_lang = get_ddg_lang(traits, params['searxng_locale'])
+
+    It might confuse, but the ``l`` value of the cookie is what SearXNG calls
+    the *region*:
+
+    .. code:: python
+
+        # !ddi paris :es-AR --> {'ad': 'es_AR', 'ah': 'ar-es', 'l': 'ar-es'}
+        params['cookies']['ad'] = eng_lang
+        params['cookies']['ah'] = eng_region
+        params['cookies']['l'] = eng_region
+
+    .. hint::
+
+       `DDG-lite <https://lite.duckduckgo.com/lite>`__ does not offer a language
+       selection to the user, only a region can be selected by the user
+       (``eng_region`` from the example above).  DDG-lite stores the selected
+       region in a cookie::
+
+         params['cookies']['kl'] = eng_region  # 'ar-es'
+
+    """
+    return eng_traits.custom['lang_region'].get(sxng_locale, eng_traits.get_language(sxng_locale, default))
+
+
+ddg_reg_map = {
+    'tw-tzh': 'zh_TW',
+    'hk-tzh': 'zh_HK',
+    'ct-ca': 'skip',  # ct-ca and es-ca both map to ca_ES
+    'es-ca': 'ca_ES',
+    'id-en': 'id_ID',
+    'no-no': 'nb_NO',
+    'jp-jp': 'ja_JP',
+    'kr-kr': 'ko_KR',
+    'xa-ar': 'ar_SA',
+    'sl-sl': 'sl_SI',
+    'th-en': 'th_TH',
+    'vn-en': 'vi_VN',
+}
+
+ddg_lang_map = {
+    # use ar --> ar_EG (Egypt's arabic)
+    "ar_DZ": 'lang_region',
+    "ar_JO": 'lang_region',
+    "ar_SA": 'lang_region',
+    # use bn --> bn_BD
+    'bn_IN': 'lang_region',
+    # use de --> de_DE
+    'de_CH': 'lang_region',
+    # use en --> en_US,
+    'en_AU': 'lang_region',
+    'en_CA': 'lang_region',
+    'en_GB': 'lang_region',
+    # Esperanto
+    'eo_XX': 'eo',
+    # use es --> es_ES,
+    'es_AR': 'lang_region',
+    'es_CL': 'lang_region',
+    'es_CO': 'lang_region',
+    'es_CR': 'lang_region',
+    'es_EC': 'lang_region',
+    'es_MX': 'lang_region',
+    'es_PE': 'lang_region',
+    'es_UY': 'lang_region',
+    'es_VE': 'lang_region',
+    # use fr --> rf_FR
+    'fr_CA': 'lang_region',
+    'fr_CH': 'lang_region',
+    'fr_BE': 'lang_region',
+    # use nl --> nl_NL
+    'nl_BE': 'lang_region',
+    # use pt --> pt_PT
+    'pt_BR': 'lang_region',
+    # skip these languages
+    'od_IN': 'skip',
+    'io_XX': 'skip',
+    'tokipona_XX': 'skip',
+}
 
 
 def request(query, params):
 
+    eng_region = traits.get_region(params['searxng_locale'], traits.all_locale)
+    # eng_lang = get_ddg_lang(traits, params['searxng_locale'])
+
     params['url'] = url
     params['method'] = 'POST'
-
     params['data']['q'] = query
 
     # The API is not documented, so we do some reverse engineering and emulate
@@ -90,23 +224,19 @@ def request(query, params):
         params['data']['s'] = offset
         params['data']['dc'] = offset + 1
 
+    # request needs a vqd argument
+    params['data']['vqd'] = get_vqd(query, params["headers"])
+
     # initial page does not have additional data in the input form
     if params['pageno'] > 1:
-        # request the second page (and more pages) needs 'o' and 'api' arguments
-        params['data']['o'] = 'json'
-        params['data']['api'] = 'd.js'
 
-    # initial page does not have additional data in the input form
-    if params['pageno'] > 2:
-        # request the third page (and more pages) some more arguments
-        params['data']['nextParams'] = ''
-        params['data']['v'] = ''
-        params['data']['vqd'] = ''
+        params['data']['o'] = form_data.get('o', 'json')
+        params['data']['api'] = form_data.get('api', 'd.js')
+        params['data']['nextParams'] = form_data.get('nextParams', '')
+        params['data']['v'] = form_data.get('v', 'l')
 
-    region_code = get_region_code(params['language'], supported_languages)
-    if region_code:
-        params['data']['kl'] = region_code
-        params['cookies']['kl'] = region_code
+    params['data']['kl'] = eng_region
+    params['cookies']['kl'] = eng_region
 
     params['data']['df'] = ''
     if params['time_range'] in time_range_dict:
@@ -118,26 +248,40 @@ def request(query, params):
     return params
 
 
-# get response from search-request
 def response(resp):
 
-    headers_ping = dict_subset(resp.request.headers, ['User-Agent', 'Accept-Encoding', 'Accept', 'Cookie'])
-    network.get(url_ping, headers=headers_ping)
-
     if resp.status_code == 303:
         return []
 
     results = []
-    doc = html.fromstring(resp.text)
+    doc = lxml.html.fromstring(resp.text)
 
     result_table = eval_xpath(doc, '//html/body/form/div[@class="filters"]/table')
-    if not len(result_table) >= 3:
+
+    if len(result_table) == 2:
+        # some locales (at least China) does not have a "next page" button and
+        # the layout of the HTML tables is different.
+        result_table = result_table[1]
+    elif not len(result_table) >= 3:
         # no more results
         return []
-    result_table = result_table[2]
+    else:
+        result_table = result_table[2]
+        # update form data from response
+        form = eval_xpath(doc, '//html/body/form/div[@class="filters"]/table//input/..')
+        if len(form):
+
+            form = form[0]
+            form_data['v'] = eval_xpath(form, '//input[@name="v"]/@value')[0]
+            form_data['api'] = eval_xpath(form, '//input[@name="api"]/@value')[0]
+            form_data['o'] = eval_xpath(form, '//input[@name="o"]/@value')[0]
+            logger.debug('form_data: %s', form_data)
+
+            value = eval_xpath(form, '//input[@name="vqd"]/@value')[0]
+            query = resp.search_params['data']['q']
+            cache_vqd(query, value)
 
     tr_rows = eval_xpath(result_table, './/tr')
-
     # In the last <tr> is the form of the 'previous/next page' links
     tr_rows = tr_rows[:-1]
 
@@ -174,32 +318,35 @@ def response(resp):
     return results
 
 
-# get supported languages from their site
-def _fetch_supported_languages(resp):
-
-    # response is a js file with regions as an embedded object
-    response_page = resp.text
-    response_page = response_page[response_page.find('regions:{') + 8 :]
-    response_page = response_page[: response_page.find('}') + 1]
-
-    regions_json = json.loads(response_page)
-    supported_languages = map((lambda x: x[3:] + '-' + x[:2].upper()), regions_json.keys())
+def fetch_traits(engine_traits: EngineTraits):
+    """Fetch languages & regions from DuckDuckGo.
 
-    return list(supported_languages)
+    SearXNG's ``all`` locale maps DuckDuckGo's "Alle regions" (``wt-wt``).
+    DuckDuckGo's language "Browsers prefered language" (``wt_WT``) makes no
+    sense in a SearXNG request since SearXNG's ``all`` will not add a
+    ``Accept-Language`` HTTP header.  The value in ``engine_traits.all_locale``
+    is ``wt-wt`` (the region).
 
+    Beside regions DuckDuckGo also defines its lanaguages by region codes.  By
+    example these are the english languages in DuckDuckGo:
 
-def fetch_traits(engine_traits: EngineTraits):
-    """Fetch regions from DuckDuckGo."""
-    # pylint: disable=import-outside-toplevel
+    - en_US
+    - en_AU
+    - en_CA
+    - en_GB
 
-    engine_traits.data_type = 'supported_languages'  # deprecated
+    The function :py:obj:`get_ddg_lang` evaluates DuckDuckGo's language from
+    SearXNG's locale.
 
-    import babel
-    from searx.locales import region_tag
+    """
+    # pylint: disable=too-many-branches, too-many-statements
+    # fetch regions
 
     engine_traits.all_locale = 'wt-wt'
 
-    resp = network.get('https://duckduckgo.com/util/u588.js')
+    # updated from u588 to u661 / should be updated automatically?
+    resp = network.get('https://duckduckgo.com/util/u661.js')
+
     if not resp.ok:
         print("ERROR: response from DuckDuckGo is not OK.")
 
@@ -208,28 +355,13 @@ def fetch_traits(engine_traits: EngineTraits):
     pos = js_code.find('}') + 1
     regions = json.loads(js_code[:pos])
 
-    reg_map = {
-        'tw-tzh': 'zh_TW',
-        'hk-tzh': 'zh_HK',
-        'ct-ca': 'skip',  # ct-ca and es-ca both map to ca_ES
-        'es-ca': 'ca_ES',
-        'id-en': 'id_ID',
-        'no-no': 'nb_NO',
-        'jp-jp': 'ja_JP',
-        'kr-kr': 'ko_KR',
-        'xa-ar': 'ar_SA',
-        'sl-sl': 'sl_SI',
-        'th-en': 'th_TH',
-        'vn-en': 'vi_VN',
-    }
-
     for eng_tag, name in regions.items():
 
         if eng_tag == 'wt-wt':
             engine_traits.all_locale = 'wt-wt'
             continue
 
-        region = reg_map.get(eng_tag)
+        region = ddg_reg_map.get(eng_tag)
         if region == 'skip':
             continue
 
@@ -238,7 +370,7 @@ def fetch_traits(engine_traits: EngineTraits):
             region = eng_lang + '_' + eng_territory.upper()
 
         try:
-            sxng_tag = region_tag(babel.Locale.parse(region))
+            sxng_tag = locales.region_tag(babel.Locale.parse(region))
         except babel.UnknownLocaleError:
             print("ERROR: %s (%s) -> %s is unknown by babel" % (name, eng_tag, region))
             continue
@@ -249,3 +381,42 @@ def fetch_traits(engine_traits: EngineTraits):
                 print("CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, eng_tag))
             continue
         engine_traits.regions[sxng_tag] = eng_tag
+
+    # fetch languages
+
+    engine_traits.custom['lang_region'] = {}
+
+    pos = resp.text.find('languages:{') + 10
+    js_code = resp.text[pos:]
+    pos = js_code.find('}') + 1
+    js_code = '{"' + js_code[1:pos].replace(':', '":').replace(',', ',"')
+    languages = json.loads(js_code)
+
+    for eng_lang, name in languages.items():
+
+        if eng_lang == 'wt_WT':
+            continue
+
+        babel_tag = ddg_lang_map.get(eng_lang, eng_lang)
+        if babel_tag == 'skip':
+            continue
+
+        try:
+
+            if babel_tag == 'lang_region':
+                sxng_tag = locales.region_tag(babel.Locale.parse(eng_lang))
+                engine_traits.custom['lang_region'][sxng_tag] = eng_lang
+                continue
+
+            sxng_tag = locales.language_tag(babel.Locale.parse(babel_tag))
+
+        except babel.UnknownLocaleError:
+            print("ERROR: language %s (%s) is unknown by babel" % (name, eng_lang))
+            continue
+
+        conflict = engine_traits.languages.get(sxng_tag)
+        if conflict:
+            if conflict != eng_lang:
+                print("CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, eng_lang))
+            continue
+        engine_traits.languages[sxng_tag] = eng_lang