diff options
author | Markus Heiser <markus.heiser@darmarIT.de> | 2022-02-20 18:29:50 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-02-20 18:29:50 +0100 |
commit | 36aee70c247fe347c69abb17ec3bdc31781204c6 (patch) | |
tree | ee67e8017c0b1c71af03981d1a78610df9864bbe | |
parent | 883e9c1ddd1cf2af0dc6e9b82885219150b9e849 (diff) | |
parent | c2d9c93b3baf365c42a8ba93d965d404dfb5741d (diff) | |
download | searxng-36aee70c247fe347c69abb17ec3bdc31781204c6.tar.gz searxng-36aee70c247fe347c69abb17ec3bdc31781204c6.zip |
Merge pull request #910 from tiekoetter/fix-909
[fix] google images engine: Fix 'scrap_img_by_id' function
-rw-r--r-- | searx/engines/google_images.py | 65 | ||||
-rwxr-xr-x | searx/webapp.py | 2 |
2 files changed, 53 insertions, 14 deletions
diff --git a/searx/engines/google_images.py b/searx/engines/google_images.py index 2855860d8..fc192d62d 100644 --- a/searx/engines/google_images.py +++ b/searx/engines/google_images.py @@ -13,6 +13,7 @@ https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs """ +import re from urllib.parse import urlencode, unquote from lxml import html @@ -68,15 +69,55 @@ def scrap_out_thumbs(dom): return ret_val -def scrap_img_by_id(script, data_id): - """Get full image URL by data-id in parent element""" - img_url = '' - _script = script.split('\n') - for i, line in enumerate(_script): - if 'gstatic.com/images' in line and data_id in line and i + 1 < len(_script): - url_line = _script[i + 1] - img_url = url_line.split('"')[1] - img_url = unquote(img_url.replace(r'\u00', r'%')) +# [0, "-H96xjSoW5DsgM", ["https://encrypted-tbn0.gstatic.com/images?q...", 155, 324] +# , ["https://assets.cdn.moviepilot.de/files/d3bf..", 576, 1200], +_RE_JS_IMAGE_URL = re.compile( + r'"' + r'([^"]*)' # -H96xjSoW5DsgM + r'",\s*\["' + r'https://[^\.]*\.gstatic.com/images[^"]*' # https://encrypted-tbn0.gstatic.com/images?q... + r'[^\[]*\["' + r'(https?://[^"]*)' # https://assets.cdn.moviepilot.de/files/d3bf... +) + + +def parse_urls_img_from_js(dom): + + # There are two HTML script tags starting with a JS function + # 'AF_initDataCallback(...)' + # + # <script nonce="zscm+Ab/JzBk1Qd4GY6wGQ"> + # AF_initDataCallback({key: 'ds:0', hash: '1', data:[], sideChannel: {}}); + # </script> + # <script nonce="zscm+Ab/JzBk1Qd4GY6wGQ"> + # AF_initDataCallback({key: 'ds:1', hash: '2', data:[null,[[["online_chips",[["the big", + # ["https://encrypted-tbn0.gstatic.com/images?q...",null,null,true,[null,0],f + # ... + # </script> + # + # The second script contains the URLs of the images. + + # The AF_initDataCallback(..) is called with very large dictionary, that + # looks like JSON but it is not JSON since it contains JS variables and + # constants like 'null' (we can't use a JSON parser for). + # + # The alternative is to parse the entire <script> and find all image URLs by + # a regular expression. + + img_src_script = eval_xpath_getindex(dom, '//script[contains(., "AF_initDataCallback({key: ")]', 1).text + data_id_to_img_url = {} + for data_id, url in _RE_JS_IMAGE_URL.findall(img_src_script): + data_id_to_img_url[data_id] = url + return data_id_to_img_url + + +def get_img_url_by_data_id(data_id_to_img_url, img_node): + """Get full image URL by @data-id from parent element.""" + + data_id = eval_xpath_getindex(img_node, '../../../@data-id', 0) + img_url = data_id_to_img_url.get(data_id, '') + img_url = unquote(img_url.replace(r'\u00', r'%')) + return img_url @@ -123,7 +164,7 @@ def response(resp): # convert the text to dom dom = html.fromstring(resp.text) img_bas64_map = scrap_out_thumbs(dom) - img_src_script = eval_xpath_getindex(dom, '//script[contains(., "AF_initDataCallback({key: ")]', 1).text + data_id_to_img_url = parse_urls_img_from_js(dom) # parse results # @@ -178,8 +219,7 @@ def response(resp): pub_descr = extract_text(pub_nodes[0]) pub_source = extract_text(pub_nodes[1]) - img_src_id = eval_xpath_getindex(img_node, '../../../@data-id', 0) - src_url = scrap_img_by_id(img_src_script, img_src_id) + src_url = get_img_url_by_data_id(data_id_to_img_url, img_node) if not src_url: src_url = thumbnail_src @@ -190,7 +230,6 @@ def response(resp): 'content': pub_descr, 'source': pub_source, 'img_src': src_url, - # 'img_format': img_format, 'thumbnail_src': thumbnail_src, 'template': 'images.html', } diff --git a/searx/webapp.py b/searx/webapp.py index 5e05f9781..eb08d63d9 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -1132,7 +1132,7 @@ def image_proxy(): 'DNT': '1', } set_context_network_name('image_proxy') - resp, stream = http_stream(method='GET', url=url, headers=request_headers) + resp, stream = http_stream(method='GET', url=url, headers=request_headers, allow_redirects=True) content_length = resp.headers.get('Content-Length') if content_length and content_length.isdigit() and int(content_length) > maximum_size: return 'Max size', 400 |