diff options
Diffstat (limited to 'searx/engines/bing.py')
| -rw-r--r-- | searx/engines/bing.py | 258 |
1 files changed, 204 insertions, 54 deletions
diff --git a/searx/engines/bing.py b/searx/engines/bing.py index 783c0056a..0f85c7036 100644 --- a/searx/engines/bing.py +++ b/searx/engines/bing.py @@ -1,16 +1,53 @@ # SPDX-License-Identifier: AGPL-3.0-or-later # lint: pylint -"""Bing (Web) +"""This is the implementation of the Bing-WEB engine. Some of this +implementations are shared by other engines: + +- :ref:`bing images engine` +- :ref:`bing news engine` +- :ref:`bing videos engine` + +On the `preference page`_ Bing offers a lot of languages an regions (see section +'Search results languages' and 'Country/region'). However, the abundant choice +does not correspond to reality, where Bing has a full-text indexer only for a +limited number of languages. By example: you can select a language like Māori +but you never get a result in this language. + +What comes a bit closer to the truth are the `search-APIs`_ but they don`t seem +to be completely correct either (if you take a closer look you will find some +inaccuracies there too): + +- :py:obj:`searx.engines.bing.bing_traits_url` +- :py:obj:`searx.engines.bing_videos.bing_traits_url` +- :py:obj:`searx.engines.bing_images.bing_traits_url` +- :py:obj:`searx.engines.bing_news.bing_traits_url` + +.. _preference page: https://www.bing.com/account/general +.. _search-APIs: https://learn.microsoft.com/en-us/bing/search-apis/ -- https://github.com/searx/searx/issues/2019#issuecomment-648227442 """ -# pylint: disable=too-many-branches +# pylint: disable=too-many-branches, invalid-name +from typing import TYPE_CHECKING +import datetime import re -from urllib.parse import urlencode, urlparse, parse_qs +import uuid +from urllib.parse import urlencode from lxml import html -from searx.utils import eval_xpath, extract_text, eval_xpath_list, match_language, eval_xpath_getindex -from searx.network import multi_requests, Request +import babel +import babel.languages + +from searx.utils import eval_xpath, extract_text, eval_xpath_list, eval_xpath_getindex +from searx import network +from searx.locales import language_tag, region_tag +from searx.enginelib.traits import EngineTraits + +if TYPE_CHECKING: + import logging + + logger: logging.Logger + +traits: EngineTraits about = { "website": 'https://www.bing.com', @@ -21,56 +58,124 @@ about = { "results": 'HTML', } +send_accept_language_header = True +"""Bing tries to guess user's language and territory from the HTTP +Accept-Language. Optional the user can select a search-language (can be +different to the UI language) and a region (market code).""" + # engine dependent config categories = ['general', 'web'] paging = True -time_range_support = False -safesearch = False -send_accept_language_header = True -supported_languages_url = 'https://www.bing.com/account/general' -language_aliases = {} - -# search-url -base_url = 'https://www.bing.com/' +time_range_support = True +safesearch = True +safesearch_types = {2: 'STRICT', 1: 'DEMOTE', 0: 'OFF'} # cookie: ADLT=STRICT -# initial query: https://www.bing.com/search?q=foo&search=&form=QBLH -inital_query = 'search?{query}&search=&form=QBLH' +base_url = 'https://www.bing.com/search' +"""Bing (Web) search URL""" -# following queries: https://www.bing.com/search?q=foo&search=&first=11&FORM=PERE -page_query = 'search?{query}&search=&first={offset}&FORM=PERE' +bing_traits_url = 'https://learn.microsoft.com/en-us/bing/search-apis/bing-web-search/reference/market-codes' +"""Bing (Web) search API description""" def _get_offset_from_pageno(pageno): return (pageno - 1) * 10 + 1 -def request(query, params): +def set_bing_cookies(params, engine_language, engine_region, SID): + + # set cookies + # ----------- + + params['cookies']['_EDGE_V'] = '1' + + # _EDGE_S: F=1&SID=3A5253BD6BCA609509B741876AF961CA&mkt=zh-tw + _EDGE_S = [ + 'F=1', + 'SID=%s' % SID, + 'mkt=%s' % engine_region.lower(), + 'ui=%s' % engine_language.lower(), + ] + params['cookies']['_EDGE_S'] = '&'.join(_EDGE_S) + logger.debug("cookie _EDGE_S=%s", params['cookies']['_EDGE_S']) + + # "_EDGE_CD": "m=zh-tw", + + _EDGE_CD = [ # pylint: disable=invalid-name + 'm=%s' % engine_region.lower(), # search region: zh-cn + 'u=%s' % engine_language.lower(), # UI: en-us + ] + + params['cookies']['_EDGE_CD'] = '&'.join(_EDGE_CD) + ';' + logger.debug("cookie _EDGE_CD=%s", params['cookies']['_EDGE_CD']) - offset = _get_offset_from_pageno(params.get('pageno', 1)) + SRCHHPGUSR = [ # pylint: disable=invalid-name + 'SRCHLANG=%s' % engine_language, + # Trying to set ADLT cookie here seems not to have any effect, I assume + # there is some age verification by a cookie (and/or session ID) needed, + # to disable the SafeSearch. + 'ADLT=%s' % safesearch_types.get(params['safesearch'], 'DEMOTE'), + ] + params['cookies']['SRCHHPGUSR'] = '&'.join(SRCHHPGUSR) + logger.debug("cookie SRCHHPGUSR=%s", params['cookies']['SRCHHPGUSR']) + + +def request(query, params): + """Assemble a Bing-Web request.""" - # logger.debug("params['pageno'] --> %s", params.get('pageno')) - # logger.debug(" offset --> %s", offset) + engine_region = traits.get_region(params['searxng_locale'], 'en-US') + engine_language = traits.get_language(params['searxng_locale'], 'en') - search_string = page_query - if offset == 1: - search_string = inital_query + SID = uuid.uuid1().hex.upper() + CVID = uuid.uuid1().hex.upper() - if params['language'] == 'all': - lang = 'EN' - else: - lang = match_language(params['language'], supported_languages, language_aliases) + set_bing_cookies(params, engine_language, engine_region, SID) - query = 'language:{} {}'.format(lang.split('-')[0].upper(), query) + # build URL query + # --------------- - search_path = search_string.format(query=urlencode({'q': query}), offset=offset) + # query term + page = int(params.get('pageno', 1)) + query_params = { + # fmt: off + 'q': query, + 'pq': query, + 'cvid': CVID, + 'qs': 'n', + 'sp': '-1' + # fmt: on + } - if offset > 1: - referer = base_url + inital_query.format(query=urlencode({'q': query})) + # page + if page > 1: + referer = base_url + '?' + urlencode(query_params) params['headers']['Referer'] = referer logger.debug("headers.Referer --> %s", referer) - params['url'] = base_url + search_path - params['headers']['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' + query_params['first'] = _get_offset_from_pageno(page) + + if page == 2: + query_params['FORM'] = 'PERE' + elif page > 2: + query_params['FORM'] = 'PERE%s' % (page - 2) + + filters = '' + if params['time_range']: + query_params['filt'] = 'custom' + + if params['time_range'] == 'day': + filters = 'ex1:"ez1"' + elif params['time_range'] == 'week': + filters = 'ex1:"ez2"' + elif params['time_range'] == 'month': + filters = 'ex1:"ez3"' + elif params['time_range'] == 'year': + epoch_1970 = datetime.date(1970, 1, 1) + today_no = (datetime.date.today() - epoch_1970).days + filters = 'ex1:"ez5_%s_%s"' % (today_no - 365, today_no) + + params['url'] = base_url + '?' + urlencode(query_params) + if filters: + params['url'] = params['url'] + '&filters=' + filters return params @@ -107,7 +212,8 @@ def response(resp): url_cite = extract_text(eval_xpath(result, './/div[@class="b_attribution"]/cite')) # Bing can shorten the URL either at the end or in the middle of the string if ( - url_cite.startswith('https://') + url_cite + and url_cite.startswith('https://') and '…' not in url_cite and '...' not in url_cite and '›' not in url_cite @@ -127,9 +233,9 @@ def response(resp): # resolve all Bing redirections in parallel request_list = [ - Request.get(u, allow_redirects=False, headers=resp.search_params['headers']) for u in url_to_resolve + network.Request.get(u, allow_redirects=False, headers=resp.search_params['headers']) for u in url_to_resolve ] - response_list = multi_requests(request_list) + response_list = network.multi_requests(request_list) for i, redirect_response in enumerate(response_list): if not isinstance(redirect_response, Exception): results[url_to_resolve_index[i]]['url'] = redirect_response.headers['location'] @@ -157,27 +263,71 @@ def response(resp): return results -# get supported languages from their site -def _fetch_supported_languages(resp): +def fetch_traits(engine_traits: EngineTraits): + """Fetch languages and regions from Bing-Web.""" + + xpath_market_codes = '//table[1]/tbody/tr/td[3]' + # xpath_country_codes = '//table[2]/tbody/tr/td[2]' + xpath_language_codes = '//table[3]/tbody/tr/td[2]' + + _fetch_traits(engine_traits, bing_traits_url, xpath_language_codes, xpath_market_codes) + + +def _fetch_traits(engine_traits: EngineTraits, url: str, xpath_language_codes: str, xpath_market_codes: str): + + # insert alias to map from a language (zh) to a language + script (zh_Hans) + engine_traits.languages['zh'] = 'zh-hans' - lang_tags = set() + resp = network.get(url) + + if not resp.ok: + print("ERROR: response from peertube is not OK.") dom = html.fromstring(resp.text) - lang_links = eval_xpath(dom, '//div[@id="language-section"]//li') - for _li in lang_links: + map_lang = {'jp': 'ja'} + for td in eval_xpath(dom, xpath_language_codes): + eng_lang = td.text - href = eval_xpath(_li, './/@href')[0] - (_scheme, _netloc, _path, _params, query, _fragment) = urlparse(href) - query = parse_qs(query, keep_blank_values=True) + if eng_lang in ('en-gb', 'pt-br'): + # language 'en' is already in the list and a language 'en-gb' can't + # be handled in SearXNG, same with pt-br which is covered by pt-pt. + continue - # fmt: off - setlang = query.get('setlang', [None, ])[0] - # example: 'mn-Cyrl-MN' --> '['mn', 'Cyrl-MN'] - lang, nation = (setlang.split('-', maxsplit=1) + [None,])[:2] # fmt: skip - # fmt: on + babel_lang = map_lang.get(eng_lang, eng_lang).replace('-', '_') + try: + sxng_tag = language_tag(babel.Locale.parse(babel_lang)) + except babel.UnknownLocaleError: + print("ERROR: language (%s) is unknown by babel" % (eng_lang)) + continue + conflict = engine_traits.languages.get(sxng_tag) + if conflict: + if conflict != eng_lang: + print("CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, eng_lang)) + continue + engine_traits.languages[sxng_tag] = eng_lang - tag = lang + '-' + nation if nation else lang - lang_tags.add(tag) + map_region = { + 'en-ID': 'id_ID', + 'no-NO': 'nb_NO', + } - return list(lang_tags) + for td in eval_xpath(dom, xpath_market_codes): + eng_region = td.text + babel_region = map_region.get(eng_region, eng_region).replace('-', '_') + + if eng_region == 'en-WW': + engine_traits.all_locale = eng_region + continue + + try: + sxng_tag = region_tag(babel.Locale.parse(babel_region)) + except babel.UnknownLocaleError: + print("ERROR: region (%s) is unknown by babel" % (eng_region)) + continue + conflict = engine_traits.regions.get(sxng_tag) + if conflict: + if conflict != eng_region: + print("CONFLICT: babel %s --> %s, %s" % (sxng_tag, conflict, eng_region)) + continue + engine_traits.regions[sxng_tag] = eng_region |