diff options
| -rw-r--r-- | searx/engines/wikidata.py | 254 | ||||
| -rw-r--r-- | tests/unit/engines/test_wikidata.py | 38 |
2 files changed, 164 insertions, 128 deletions
diff --git a/searx/engines/wikidata.py b/searx/engines/wikidata.py index 03a58a31a..5ea2b9958 100644 --- a/searx/engines/wikidata.py +++ b/searx/engines/wikidata.py @@ -20,6 +20,7 @@ from searx.utils import match_language from json import loads from lxml.html import fromstring +from lxml import etree logger = logger.getChild('wikidata') result_count = 1 @@ -27,23 +28,23 @@ result_count = 1 # urls wikidata_host = 'https://www.wikidata.org' url_search = wikidata_host \ - + '/w/index.php?{query}' + + '/w/index.php?{query}&ns0=1' wikidata_api = wikidata_host + '/w/api.php' url_detail = wikidata_api\ + '?action=parse&format=json&{query}'\ - + '&redirects=1&prop=text%7Cdisplaytitle%7Clanglinks%7Crevid'\ - + '&disableeditsection=1&disabletidy=1&preview=1§ionpreview=1&disabletoc=1&utf8=1&formatversion=2' + + '&redirects=1&prop=text%7Cdisplaytitle%7Cparsewarnings'\ + + '&disableeditsection=1&preview=1§ionpreview=1&disabletoc=1&utf8=1&formatversion=2' url_map = 'https://www.openstreetmap.org/'\ + '?lat={latitude}&lon={longitude}&zoom={zoom}&layers=M' url_image = 'https://commons.wikimedia.org/wiki/Special:FilePath/{filename}?width=500&height=400' # xpaths +div_ids_xpath = '//div[@id]' wikidata_ids_xpath = '//ul[@class="mw-search-results"]/li//a/@href' title_xpath = '//*[contains(@class,"wikibase-title-label")]' description_xpath = '//div[contains(@class,"wikibase-entitytermsview-heading-description")]' -property_xpath = '//div[@id="{propertyid}"]' label_xpath = './/div[contains(@class,"wikibase-statementgroupview-property-label")]/a' url_xpath = './/a[contains(@class,"external free") or contains(@class, "wb-external-id")]' wikilink_xpath = './/ul[contains(@class,"wikibase-sitelinklistview-listview")]'\ @@ -56,6 +57,31 @@ language_fallback_xpath = '//sup[contains(@class,"wb-language-fallback-indicator calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]' media_xpath = value_xpath + '//div[contains(@class,"commons-media-caption")]//a' +# xpath_cache +xpath_cache = {} + + +def get_xpath(xpath_str): + result = xpath_cache.get(xpath_str, None) + if not result: + result = etree.XPath(xpath_str) + xpath_cache[xpath_str] = result + return result + + +def eval_xpath(element, xpath_str): + xpath = get_xpath(xpath_str) + return xpath(element) + + +def get_id_cache(result): + id_cache = {} + for e in eval_xpath(result, div_ids_xpath): + id = e.get('id') + if id.startswith('P'): + id_cache[id] = e + return id_cache + def request(query, params): params['url'] = url_search.format( @@ -65,8 +91,9 @@ def request(query, params): def response(resp): results = [] - html = fromstring(resp.text) - search_results = html.xpath(wikidata_ids_xpath) + htmlparser = etree.HTMLParser() + html = fromstring(resp.content.decode("utf-8"), parser=htmlparser) + search_results = eval_xpath(html, wikidata_ids_xpath) if resp.search_params['language'].split('-')[0] == 'all': language = 'en' @@ -78,13 +105,13 @@ def response(resp): wikidata_id = search_result.split('/')[-1] url = url_detail.format(query=urlencode({'page': wikidata_id, 'uselang': language})) htmlresponse = get(url) - jsonresponse = loads(htmlresponse.text) - results += getDetail(jsonresponse, wikidata_id, language, resp.search_params['language']) + jsonresponse = loads(htmlresponse.content.decode("utf-8")) + results += getDetail(jsonresponse, wikidata_id, language, resp.search_params['language'], htmlparser) return results -def getDetail(jsonresponse, wikidata_id, language, locale): +def getDetail(jsonresponse, wikidata_id, language, locale, htmlparser): results = [] urls = [] attributes = [] @@ -95,21 +122,23 @@ def getDetail(jsonresponse, wikidata_id, language, locale): if not title or not result: return results - title = fromstring(title) - for elem in title.xpath(language_fallback_xpath): + title = fromstring(title, parser=htmlparser) + for elem in eval_xpath(title, language_fallback_xpath): elem.getparent().remove(elem) - title = extract_text(title.xpath(title_xpath)) + title = extract_text(eval_xpath(title, title_xpath)) - result = fromstring(result) - for elem in result.xpath(language_fallback_xpath): + result = fromstring(result, parser=htmlparser) + for elem in eval_xpath(result, language_fallback_xpath): elem.getparent().remove(elem) - description = extract_text(result.xpath(description_xpath)) + description = extract_text(eval_xpath(result, description_xpath)) + + id_cache = get_id_cache(result) # URLS # official website - add_url(urls, result, 'P856', results=results) + add_url(urls, result, id_cache, 'P856', results=results) # wikipedia wikipedia_link_count = 0 @@ -130,30 +159,30 @@ def getDetail(jsonresponse, wikidata_id, language, locale): # if wikipedia_link_count == 0: # more wikis - add_url(urls, result, default_label='Wikivoyage (' + language + ')', link_type=language + 'wikivoyage') - add_url(urls, result, default_label='Wikiquote (' + language + ')', link_type=language + 'wikiquote') - add_url(urls, result, default_label='Wikimedia Commons', link_type='commonswiki') + add_url(urls, result, id_cache, default_label='Wikivoyage (' + language + ')', link_type=language + 'wikivoyage') + add_url(urls, result, id_cache, default_label='Wikiquote (' + language + ')', link_type=language + 'wikiquote') + add_url(urls, result, id_cache, default_label='Wikimedia Commons', link_type='commonswiki') - add_url(urls, result, 'P625', 'OpenStreetMap', link_type='geo') + add_url(urls, result, id_cache, 'P625', 'OpenStreetMap', link_type='geo') # musicbrainz - add_url(urls, result, 'P434', 'MusicBrainz', 'http://musicbrainz.org/artist/') - add_url(urls, result, 'P435', 'MusicBrainz', 'http://musicbrainz.org/work/') - add_url(urls, result, 'P436', 'MusicBrainz', 'http://musicbrainz.org/release-group/') - add_url(urls, result, 'P966', 'MusicBrainz', 'http://musicbrainz.org/label/') + add_url(urls, result, id_cache, 'P434', 'MusicBrainz', 'http://musicbrainz.org/artist/') + add_url(urls, result, id_cache, 'P435', 'MusicBrainz', 'http://musicbrainz.org/work/') + add_url(urls, result, id_cache, 'P436', 'MusicBrainz', 'http://musicbrainz.org/release-group/') + add_url(urls, result, id_cache, 'P966', 'MusicBrainz', 'http://musicbrainz.org/label/') # IMDb - add_url(urls, result, 'P345', 'IMDb', 'https://www.imdb.com/', link_type='imdb') + add_url(urls, result, id_cache, 'P345', 'IMDb', 'https://www.imdb.com/', link_type='imdb') # source code repository - add_url(urls, result, 'P1324') + add_url(urls, result, id_cache, 'P1324') # blog - add_url(urls, result, 'P1581') + add_url(urls, result, id_cache, 'P1581') # social media links - add_url(urls, result, 'P2397', 'YouTube', 'https://www.youtube.com/channel/') - add_url(urls, result, 'P1651', 'YouTube', 'https://www.youtube.com/watch?v=') - add_url(urls, result, 'P2002', 'Twitter', 'https://twitter.com/') - add_url(urls, result, 'P2013', 'Facebook', 'https://facebook.com/') - add_url(urls, result, 'P2003', 'Instagram', 'https://instagram.com/') + add_url(urls, result, id_cache, 'P2397', 'YouTube', 'https://www.youtube.com/channel/') + add_url(urls, result, id_cache, 'P1651', 'YouTube', 'https://www.youtube.com/watch?v=') + add_url(urls, result, id_cache, 'P2002', 'Twitter', 'https://twitter.com/') + add_url(urls, result, id_cache, 'P2013', 'Facebook', 'https://facebook.com/') + add_url(urls, result, id_cache, 'P2003', 'Instagram', 'https://instagram.com/') urls.append({'title': 'Wikidata', 'url': 'https://www.wikidata.org/wiki/' @@ -163,132 +192,132 @@ def getDetail(jsonresponse, wikidata_id, language, locale): # DATES # inception date - add_attribute(attributes, result, 'P571', date=True) + add_attribute(attributes, id_cache, 'P571', date=True) # dissolution date - add_attribute(attributes, result, 'P576', date=True) + add_attribute(attributes, id_cache, 'P576', date=True) # start date - add_attribute(attributes, result, 'P580', date=True) + add_attribute(attributes, id_cache, 'P580', date=True) # end date - add_attribute(attributes, result, 'P582', date=True) + add_attribute(attributes, id_cache, 'P582', date=True) # date of birth - add_attribute(attributes, result, 'P569', date=True) + add_attribute(attributes, id_cache, 'P569', date=True) # date of death - add_attribute(attributes, result, 'P570', date=True) + add_attribute(attributes, id_cache, 'P570', date=True) # date of spacecraft launch - add_attribute(attributes, result, 'P619', date=True) + add_attribute(attributes, id_cache, 'P619', date=True) # date of spacecraft landing - add_attribute(attributes, result, 'P620', date=True) + add_attribute(attributes, id_cache, 'P620', date=True) # nationality - add_attribute(attributes, result, 'P27') + add_attribute(attributes, id_cache, 'P27') # country of origin - add_attribute(attributes, result, 'P495') + add_attribute(attributes, id_cache, 'P495') # country - add_attribute(attributes, result, 'P17') + add_attribute(attributes, id_cache, 'P17') # headquarters - add_attribute(attributes, result, 'Q180') + add_attribute(attributes, id_cache, 'Q180') # PLACES # capital - add_attribute(attributes, result, 'P36', trim=True) + add_attribute(attributes, id_cache, 'P36', trim=True) # head of state - add_attribute(attributes, result, 'P35', trim=True) + add_attribute(attributes, id_cache, 'P35', trim=True) # head of government - add_attribute(attributes, result, 'P6', trim=True) + add_attribute(attributes, id_cache, 'P6', trim=True) # type of government - add_attribute(attributes, result, 'P122') + add_attribute(attributes, id_cache, 'P122') # official language - add_attribute(attributes, result, 'P37') + add_attribute(attributes, id_cache, 'P37') # population - add_attribute(attributes, result, 'P1082', trim=True) + add_attribute(attributes, id_cache, 'P1082', trim=True) # area - add_attribute(attributes, result, 'P2046') + add_attribute(attributes, id_cache, 'P2046') # currency - add_attribute(attributes, result, 'P38', trim=True) + add_attribute(attributes, id_cache, 'P38', trim=True) # heigth (building) - add_attribute(attributes, result, 'P2048') + add_attribute(attributes, id_cache, 'P2048') # MEDIA # platform (videogames) - add_attribute(attributes, result, 'P400') + add_attribute(attributes, id_cache, 'P400') # author - add_attribute(attributes, result, 'P50') + add_attribute(attributes, id_cache, 'P50') # creator - add_attribute(attributes, result, 'P170') + add_attribute(attributes, id_cache, 'P170') # director - add_attribute(attributes, result, 'P57') + add_attribute(attributes, id_cache, 'P57') # performer - add_attribute(attributes, result, 'P175') + add_attribute(attributes, id_cache, 'P175') # developer - add_attribute(attributes, result, 'P178') + add_attribute(attributes, id_cache, 'P178') # producer - add_attribute(attributes, result, 'P162') + add_attribute(attributes, id_cache, 'P162') # manufacturer - add_attribute(attributes, result, 'P176') + add_attribute(attributes, id_cache, 'P176') # screenwriter - add_attribute(attributes, result, 'P58') + add_attribute(attributes, id_cache, 'P58') # production company - add_attribute(attributes, result, 'P272') + add_attribute(attributes, id_cache, 'P272') # record label - add_attribute(attributes, result, 'P264') + add_attribute(attributes, id_cache, 'P264') # publisher - add_attribute(attributes, result, 'P123') + add_attribute(attributes, id_cache, 'P123') # original network - add_attribute(attributes, result, 'P449') + add_attribute(attributes, id_cache, 'P449') # distributor - add_attribute(attributes, result, 'P750') + add_attribute(attributes, id_cache, 'P750') # composer - add_attribute(attributes, result, 'P86') + add_attribute(attributes, id_cache, 'P86') # publication date - add_attribute(attributes, result, 'P577', date=True) + add_attribute(attributes, id_cache, 'P577', date=True) # genre - add_attribute(attributes, result, 'P136') + add_attribute(attributes, id_cache, 'P136') # original language - add_attribute(attributes, result, 'P364') + add_attribute(attributes, id_cache, 'P364') # isbn - add_attribute(attributes, result, 'Q33057') + add_attribute(attributes, id_cache, 'Q33057') # software license - add_attribute(attributes, result, 'P275') + add_attribute(attributes, id_cache, 'P275') # programming language - add_attribute(attributes, result, 'P277') + add_attribute(attributes, id_cache, 'P277') # version - add_attribute(attributes, result, 'P348', trim=True) + add_attribute(attributes, id_cache, 'P348', trim=True) # narrative location - add_attribute(attributes, result, 'P840') + add_attribute(attributes, id_cache, 'P840') # LANGUAGES # number of speakers - add_attribute(attributes, result, 'P1098') + add_attribute(attributes, id_cache, 'P1098') # writing system - add_attribute(attributes, result, 'P282') + add_attribute(attributes, id_cache, 'P282') # regulatory body - add_attribute(attributes, result, 'P1018') + add_attribute(attributes, id_cache, 'P1018') # language code - add_attribute(attributes, result, 'P218') + add_attribute(attributes, id_cache, 'P218') # OTHER # ceo - add_attribute(attributes, result, 'P169', trim=True) + add_attribute(attributes, id_cache, 'P169', trim=True) # founder - add_attribute(attributes, result, 'P112') + add_attribute(attributes, id_cache, 'P112') # legal form (company/organization) - add_attribute(attributes, result, 'P1454') + add_attribute(attributes, id_cache, 'P1454') # operator - add_attribute(attributes, result, 'P137') + add_attribute(attributes, id_cache, 'P137') # crew members (tripulation) - add_attribute(attributes, result, 'P1029') + add_attribute(attributes, id_cache, 'P1029') # taxon - add_attribute(attributes, result, 'P225') + add_attribute(attributes, id_cache, 'P225') # chemical formula - add_attribute(attributes, result, 'P274') + add_attribute(attributes, id_cache, 'P274') # winner (sports/contests) - add_attribute(attributes, result, 'P1346') + add_attribute(attributes, id_cache, 'P1346') # number of deaths - add_attribute(attributes, result, 'P1120') + add_attribute(attributes, id_cache, 'P1120') # currency code - add_attribute(attributes, result, 'P498') + add_attribute(attributes, id_cache, 'P498') - image = add_image(result) + image = add_image(id_cache) if len(attributes) == 0 and len(urls) == 2 and len(description) == 0: results.append({ @@ -310,43 +339,42 @@ def getDetail(jsonresponse, wikidata_id, language, locale): # only returns first match -def add_image(result): +def add_image(id_cache): # P15: route map, P242: locator map, P154: logo, P18: image, P242: map, P41: flag, P2716: collage, P2910: icon property_ids = ['P15', 'P242', 'P154', 'P18', 'P242', 'P41', 'P2716', 'P2910'] for property_id in property_ids: - image = result.xpath(property_xpath.replace('{propertyid}', property_id)) - if image: - image_name = image[0].xpath(media_xpath) + image = id_cache.get(property_id, None) + if image is not None: + image_name = eval_xpath(image, media_xpath) image_src = url_image.replace('{filename}', extract_text(image_name[0])) return image_src # setting trim will only returned high ranked rows OR the first row -def add_attribute(attributes, result, property_id, default_label=None, date=False, trim=False): - attribute = result.xpath(property_xpath.replace('{propertyid}', property_id)) - if attribute: +def add_attribute(attributes, id_cache, property_id, default_label=None, date=False, trim=False): + attribute = id_cache.get(property_id, None) + if attribute is not None: if default_label: label = default_label else: - label = extract_text(attribute[0].xpath(label_xpath)) + label = extract_text(eval_xpath(attribute, label_xpath)) label = label[0].upper() + label[1:] if date: trim = True # remove calendar name - calendar_name = attribute[0].xpath(calendar_name_xpath) + calendar_name = eval_xpath(attribute, calendar_name_xpath) for calendar in calendar_name: calendar.getparent().remove(calendar) concat_values = "" values = [] first_value = None - for row in attribute[0].xpath(property_row_xpath): - if not first_value or not trim or row.xpath(preferred_rank_xpath): - - value = row.xpath(value_xpath) + for row in eval_xpath(attribute, property_row_xpath): + if not first_value or not trim or eval_xpath(row, preferred_rank_xpath): + value = eval_xpath(row, value_xpath) if not value: continue value = extract_text(value) @@ -369,18 +397,18 @@ def add_attribute(attributes, result, property_id, default_label=None, date=Fals # requires property_id unless it's a wiki link (defined in link_type) -def add_url(urls, result, property_id=None, default_label=None, url_prefix=None, results=None, link_type=None): +def add_url(urls, result, id_cache, property_id=None, default_label=None, url_prefix=None, results=None, + link_type=None): links = [] # wiki links don't have property in wikidata page if link_type and 'wiki' in link_type: links.append(get_wikilink(result, link_type)) else: - dom_element = result.xpath(property_xpath.replace('{propertyid}', property_id)) - if dom_element: - dom_element = dom_element[0] + dom_element = id_cache.get(property_id, None) + if dom_element is not None: if not default_label: - label = extract_text(dom_element.xpath(label_xpath)) + label = extract_text(eval_xpath(dom_element, label_xpath)) label = label[0].upper() + label[1:] if link_type == 'geo': @@ -390,7 +418,7 @@ def add_url(urls, result, property_id=None, default_label=None, url_prefix=None, links.append(get_imdblink(dom_element, url_prefix)) else: - url_results = dom_element.xpath(url_xpath) + url_results = eval_xpath(dom_element, url_xpath) for link in url_results: if link is not None: if url_prefix: @@ -410,7 +438,7 @@ def add_url(urls, result, property_id=None, default_label=None, url_prefix=None, def get_imdblink(result, url_prefix): - imdb_id = result.xpath(value_xpath) + imdb_id = eval_xpath(result, value_xpath) if imdb_id: imdb_id = extract_text(imdb_id) id_prefix = imdb_id[:2] @@ -430,7 +458,7 @@ def get_imdblink(result, url_prefix): def get_geolink(result): - coordinates = result.xpath(value_xpath) + coordinates = eval_xpath(result, value_xpath) if not coordinates: return None coordinates = extract_text(coordinates[0]) @@ -477,7 +505,7 @@ def get_geolink(result): def get_wikilink(result, wikiid): - url = result.xpath(wikilink_xpath.replace('{wikiid}', wikiid)) + url = eval_xpath(result, wikilink_xpath.replace('{wikiid}', wikiid)) if not url: return None url = url[0] diff --git a/tests/unit/engines/test_wikidata.py b/tests/unit/engines/test_wikidata.py index 347defb71..48be17bb4 100644 --- a/tests/unit/engines/test_wikidata.py +++ b/tests/unit/engines/test_wikidata.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- from lxml.html import fromstring +from lxml import etree from collections import defaultdict import mock from searx.engines import wikidata @@ -30,12 +31,12 @@ class TestWikidataEngine(SearxTestCase): wikidata.supported_languages = ['en', 'es'] wikidata.language_aliases = {} - response = mock.Mock(text='<html></html>', search_params={"language": "en"}) + response = mock.Mock(content='<html></html>'.encode("utf-8"), search_params={"language": "en"}) self.assertEqual(wikidata.response(response), []) def test_getDetail(self): response = {} - results = wikidata.getDetail(response, "Q123", "en", "en-US") + results = wikidata.getDetail(response, "Q123", "en", "en-US", etree.HTMLParser()) self.assertEqual(results, []) title_html = '<div><div class="wikibase-title-label">Test</div></div>' @@ -52,7 +53,7 @@ class TestWikidataEngine(SearxTestCase): """ response = {"parse": {"displaytitle": title_html, "text": html}} - results = wikidata.getDetail(response, "Q123", "en", "en-US") + results = wikidata.getDetail(response, "Q123", "en", "en-US", etree.HTMLParser()) self.assertEqual(len(results), 1) self.assertEqual(results[0]['url'], 'https://en.wikipedia.org/wiki/Test') @@ -92,7 +93,7 @@ class TestWikidataEngine(SearxTestCase): """ response = {"parse": {"displaytitle": title_html, "text": html}} - results = wikidata.getDetail(response, "Q123", "yua", "yua_MX") + results = wikidata.getDetail(response, "Q123", "yua", "yua_MX", etree.HTMLParser()) self.assertEqual(len(results), 2) self.assertEqual(results[0]['title'], 'Official website') self.assertEqual(results[0]['url'], 'https://officialsite.com') @@ -139,8 +140,8 @@ class TestWikidataEngine(SearxTestCase): </div> """ html_etree = fromstring(html) - - image_src = wikidata.add_image(html_etree) + id_cache = wikidata.get_id_cache(html_etree) + image_src = wikidata.add_image(id_cache) self.assertEqual(image_src, "https://commons.wikimedia.org/wiki/Special:FilePath/image.png?width=500&height=400") @@ -197,8 +198,9 @@ class TestWikidataEngine(SearxTestCase): </div> """ html_etree = fromstring(html) + id_cache = wikidata.get_id_cache(html_etree) - image_src = wikidata.add_image(html_etree) + image_src = wikidata.add_image(id_cache) self.assertEqual(image_src, "https://commons.wikimedia.org/wiki/Special:FilePath/logo.png?width=500&height=400") @@ -232,11 +234,12 @@ class TestWikidataEngine(SearxTestCase): """ attributes = [] html_etree = fromstring(html) + id_cache = wikidata.get_id_cache(html_etree) - wikidata.add_attribute(attributes, html_etree, "Fail") + wikidata.add_attribute(attributes, id_cache, "Fail") self.assertEqual(attributes, []) - wikidata.add_attribute(attributes, html_etree, "P27") + wikidata.add_attribute(attributes, id_cache, "P27") self.assertEqual(len(attributes), 1) self.assertEqual(attributes[0]["label"], "Country of citizenship") self.assertEqual(attributes[0]["value"], "United Kingdom") @@ -271,7 +274,8 @@ class TestWikidataEngine(SearxTestCase): """ attributes = [] html_etree = fromstring(html) - wikidata.add_attribute(attributes, html_etree, "P569", date=True) + id_cache = wikidata.get_id_cache(html_etree) + wikidata.add_attribute(attributes, id_cache, "P569", date=True) self.assertEqual(len(attributes), 1) self.assertEqual(attributes[0]["label"], "Date of birth") self.assertEqual(attributes[0]["value"], "27 January 1832") @@ -319,14 +323,16 @@ class TestWikidataEngine(SearxTestCase): """ attributes = [] html_etree = fromstring(html) - wikidata.add_attribute(attributes, html_etree, "P6") + id_cache = wikidata.get_id_cache(html_etree) + wikidata.add_attribute(attributes, id_cache, "P6") self.assertEqual(len(attributes), 1) self.assertEqual(attributes[0]["label"], "Head of government") self.assertEqual(attributes[0]["value"], "Old Prime Minister, Actual Prime Minister") attributes = [] html_etree = fromstring(html) - wikidata.add_attribute(attributes, html_etree, "P6", trim=True) + id_cache = wikidata.get_id_cache(html_etree) + wikidata.add_attribute(attributes, id_cache, "P6", trim=True) self.assertEqual(len(attributes), 1) self.assertEqual(attributes[0]["value"], "Actual Prime Minister") @@ -357,12 +363,13 @@ class TestWikidataEngine(SearxTestCase): """ urls = [] html_etree = fromstring(html) - wikidata.add_url(urls, html_etree, 'P856') + id_cache = wikidata.get_id_cache(html_etree) + wikidata.add_url(urls, html_etree, id_cache, 'P856') self.assertEquals(len(urls), 1) self.assertIn({'title': 'Official website', 'url': 'https://searx.me/'}, urls) urls = [] results = [] - wikidata.add_url(urls, html_etree, 'P856', 'custom label', results=results) + wikidata.add_url(urls, html_etree, id_cache, 'P856', 'custom label', results=results) self.assertEquals(len(urls), 1) self.assertEquals(len(results), 1) self.assertIn({'title': 'custom label', 'url': 'https://searx.me/'}, urls) @@ -405,7 +412,8 @@ class TestWikidataEngine(SearxTestCase): """ urls = [] html_etree = fromstring(html) - wikidata.add_url(urls, html_etree, 'P856') + id_cache = wikidata.get_id_cache(html_etree) + wikidata.add_url(urls, html_etree, id_cache, 'P856') self.assertEquals(len(urls), 2) self.assertIn({'title': 'Official website', 'url': 'http://www.worldofwarcraft.com'}, urls) self.assertIn({'title': 'Official website', 'url': 'http://eu.battle.net/wow/en/'}, urls) |