diff options
| -rw-r--r-- | Makefile | 3 | ||||
| -rw-r--r-- | searx/engines/duckduckgo.py | 18 | ||||
| -rw-r--r-- | searx/engines/mediathekviewweb.py | 68 | ||||
| -rw-r--r-- | searx/poolrequests.py | 12 | ||||
| -rw-r--r-- | searx/search/processors/online.py | 2 | ||||
| -rw-r--r-- | searx/settings.yml | 4 |
6 files changed, 92 insertions, 15 deletions
@@ -193,7 +193,8 @@ PYLINT_FILES=\ searx/engines/google.py \ searx/engines/google_news.py \ searx/engines/google_videos.py \ - searx/engines/google_images.py + searx/engines/google_images.py \ + searx/engines/mediathekviewweb.py test.pylint: pyenvinstall $(call cmd,pylint,$(PYLINT_FILES)) diff --git a/searx/engines/duckduckgo.py b/searx/engines/duckduckgo.py index 7f1378264..638f1211b 100644 --- a/searx/engines/duckduckgo.py +++ b/searx/engines/duckduckgo.py @@ -5,7 +5,8 @@ from lxml.html import fromstring from json import loads -from searx.utils import extract_text, match_language, eval_xpath +from searx.utils import extract_text, match_language, eval_xpath, dict_subset +from searx.poolrequests import get # about about = { @@ -35,6 +36,7 @@ language_aliases = { # search-url url = 'https://html.duckduckgo.com/html' +url_ping = 'https://duckduckgo.com/t/sl_h' time_range_dict = {'day': 'd', 'week': 'w', 'month': 'm'} @@ -65,27 +67,27 @@ def request(query, params): params['url'] = url params['method'] = 'POST' - params['data']['b'] = '' params['data']['q'] = query - params['data']['df'] = '' + params['data']['b'] = '' region_code = get_region_code(params['language'], supported_languages) if region_code: params['data']['kl'] = region_code params['cookies']['kl'] = region_code - if params['time_range'] in time_range_dict: - params['data']['df'] = time_range_dict[params['time_range']] + params['data']['df'] = time_range_dict.get(params['time_range'], '') return params # get response from search-request def response(resp): - results = [] + # ping + headers_ping = dict_subset(resp.request.headers, ['User-Agent', 'Accept-Encoding', 'Accept', 'Cookie']) + get(url_ping, headers=headers_ping) + # parse the response + results = [] doc = fromstring(resp.text) - - # parse results for i, r in enumerate(eval_xpath(doc, result_xpath)): if i >= 30: break diff --git a/searx/engines/mediathekviewweb.py b/searx/engines/mediathekviewweb.py new file mode 100644 index 000000000..fa442c937 --- /dev/null +++ b/searx/engines/mediathekviewweb.py @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +"""MediathekViewWeb (API) + +""" + +# pylint: disable=missing-function-docstring + +import datetime +from json import loads, dumps + +about = { + "website": 'https://mediathekviewweb.de/', + "wikidata_id": 'Q27877380', + "official_api_documentation": 'https://gist.github.com/bagbag/a2888478d27de0e989cf777f81fb33de', + "use_official_api": True, + "require_api_key": False, + "results": 'JSON', +} + +categories = ['videos'] +paging = True +time_range_support = False +safesearch = False + +def request(query, params): + + params['url'] = 'https://mediathekviewweb.de/api/query' + params['method'] = 'POST' + params['headers']['Content-type'] = 'text/plain' + params['data'] = dumps({ + 'queries' : [ + { + 'fields' : [ + 'title', + 'topic', + ], + 'query' : query + }, + ], + 'sortBy' : 'timestamp', + 'sortOrder' : 'desc', + 'future' : True, + 'offset' : (params['pageno'] - 1 )* 10, + 'size' : 10 + }) + return params + +def response(resp): + + resp = loads(resp.text) + + mwv_result = resp['result'] + mwv_result_list = mwv_result['results'] + + results = [] + + for item in mwv_result_list: + + item['hms'] = str(datetime.timedelta(seconds=item['duration'])) + + results.append({ + 'url' : item['url_video_hd'], + 'title' : "%(channel)s: %(title)s (%(hms)s)" % item, + 'length' : item['hms'], + 'content' : "%(description)s" % item, + }) + + return results diff --git a/searx/poolrequests.py b/searx/poolrequests.py index 25a6baed9..8b8681437 100644 --- a/searx/poolrequests.py +++ b/searx/poolrequests.py @@ -1,7 +1,7 @@ import sys from time import time from itertools import cycle -from threading import RLock, local +from threading import local import requests @@ -88,10 +88,12 @@ class SessionSinglePool(requests.Session): super().__init__() # reuse the same adapters - with RLock(): - self.adapters.clear() - self.mount('https://', next(https_adapters)) - self.mount('http://', next(http_adapters)) + self.adapters.clear() + + https_adapter = threadLocal.__dict__.setdefault('https_adapter', next(https_adapters)) + http_adapter = threadLocal.__dict__.setdefault('http_adapter', next(http_adapters)) + self.mount('https://', https_adapter) + self.mount('http://', http_adapter) def close(self): """Call super, but clear adapters since there are managed globaly""" diff --git a/searx/search/processors/online.py b/searx/search/processors/online.py index d79edd542..0cc175e1b 100644 --- a/searx/search/processors/online.py +++ b/searx/search/processors/online.py @@ -77,7 +77,7 @@ class OnlineProcessor(EngineProcessor): soft_max_redirects = params.get('soft_max_redirects', max_redirects or 0) # raise_for_status - request_args['raise_for_httperror'] = params.get('raise_for_httperror', False) + request_args['raise_for_httperror'] = params.get('raise_for_httperror', True) # specific type of request (GET or POST) if params['method'] == 'GET': diff --git a/searx/settings.yml b/searx/settings.yml index 81a5190e9..4e926d73c 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -1242,6 +1242,10 @@ engines: categories: videos disabled : True + - name : mediathekviewweb + engine : mediathekviewweb + shortcut : mvw + # - name : yacy # engine : yacy # shortcut : ya |