diff options
| -rw-r--r-- | searx/engines/duden.py | 19 | ||||
| -rw-r--r-- | tests/unit/engines/test_duden.py | 28 |
2 files changed, 28 insertions, 19 deletions
diff --git a/searx/engines/duden.py b/searx/engines/duden.py index 881ff9d9c..444f18c1f 100644 --- a/searx/engines/duden.py +++ b/searx/engines/duden.py @@ -11,7 +11,7 @@ from lxml import html, etree import re from searx.engines.xpath import extract_text -from searx.url_utils import quote +from searx.url_utils import quote, urljoin from searx import logger categories = ['general'] @@ -20,7 +20,7 @@ language_support = False # search-url base_url = 'https://www.duden.de/' -search_url = base_url + 'suchen/dudenonline/{query}?page={offset}' +search_url = base_url + 'suchen/dudenonline/{query}?search_api_fulltext=&page={offset}' def request(query, params): @@ -35,7 +35,11 @@ def request(query, params): ''' offset = (params['pageno'] - 1) - params['url'] = search_url.format(offset=offset, query=quote(query)) + if offset == 0: + search_url_fmt = base_url + 'suchen/dudenonline/{query}' + params['url'] = search_url_fmt.format(query=quote(query)) + else: + params['url'] = search_url.format(offset=offset, query=quote(query)) return params @@ -58,12 +62,11 @@ def response(resp): logger.debug("Couldn't read number of results.") pass - for result in dom.xpath('//section[@class="wide" and not(contains(@style,"overflow:hidden"))]'): + for result in dom.xpath('//section[not(contains(@class, "essay"))]'): try: - logger.debug("running for %s" % str(result)) - link = result.xpath('.//h2/a')[0] - url = link.attrib.get('href') - title = result.xpath('string(.//h2/a)') + url = result.xpath('.//h2/a')[0].get('href') + url = urljoin(base_url, url) + title = result.xpath('string(.//h2/a)').strip() content = extract_text(result.xpath('.//p')) # append result results.append({'url': url, diff --git a/tests/unit/engines/test_duden.py b/tests/unit/engines/test_duden.py index d9bbfef8b..52fc513d0 100644 --- a/tests/unit/engines/test_duden.py +++ b/tests/unit/engines/test_duden.py @@ -10,24 +10,30 @@ class TestDudenEngine(SearxTestCase): def test_request(self): query = 'Haus' dic = defaultdict(dict) - dic['pageno'] = 1 - params = duden.request(query, dic) - self.assertTrue('url' in params) - self.assertTrue(query in params['url']) - self.assertTrue('duden.de' in params['url']) + data = [ + [1, 'https://www.duden.de/suchen/dudenonline/Haus'], + [2, 'https://www.duden.de/suchen/dudenonline/Haus?search_api_fulltext=&page=1'] + ] + for page_no, exp_res in data: + dic['pageno'] = page_no + params = duden.request(query, dic) + self.assertTrue('url' in params) + self.assertTrue(query in params['url']) + self.assertTrue('duden.de' in params['url']) + self.assertEqual(params['url'], exp_res) def test_response(self): resp = mock.Mock(text='<html></html>') self.assertEqual(duden.response(resp), []) html = """ - <section class="wide"> - <h2><a href="https://this.is.the.url/" class="hidden-link"><strong>This is the title</strong> also here</a></h2> - <p>This is the <strong>content</strong></p> - <a href="https://this.is.the.url/">Zum vollständigen Artikel</a> + <section class="vignette"> + <h2"> <a href="/rechtschreibung/Haus"> + <strong>This is the title also here</strong> + </a> </h2> + <p>This is the content</p> </section> """ - resp = mock.Mock(text=html) results = duden.response(resp) @@ -36,6 +42,6 @@ class TestDudenEngine(SearxTestCase): # testing result (dictionary entry) r = results[0] - self.assertEqual(r['url'], 'https://this.is.the.url/') + self.assertEqual(r['url'], 'https://www.duden.de/rechtschreibung/Haus') self.assertEqual(r['title'], 'This is the title also here') self.assertEqual(r['content'], 'This is the content') |