summaryrefslogtreecommitdiff
path: root/tests/unit/engines/test_xpath.py
blob: 24f14127b227d3785ec624927040a5f09bb7fa53 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
# SPDX-License-Identifier: AGPL-3.0-or-later
# pylint: disable=missing-module-docstring

from collections import defaultdict
import mock
from searx.engines import xpath
from tests import SearxTestCase


class TestXpathEngine(SearxTestCase):
    def test_request(self):
        xpath.search_url = 'https://url.com/{query}'
        xpath.categories = []
        xpath.paging = False
        query = 'test_query'
        dicto = defaultdict(dict)
        params = xpath.request(query, dicto)
        self.assertIn('url', params)
        self.assertEquals('https://url.com/test_query', params['url'])

        xpath.search_url = 'https://url.com/q={query}&p={pageno}'
        xpath.paging = True
        query = 'test_query'
        dicto = defaultdict(dict)
        dicto['pageno'] = 1
        params = xpath.request(query, dicto)
        self.assertIn('url', params)
        self.assertEquals('https://url.com/q=test_query&p=1', params['url'])

    def test_response(self):
        # without results_xpath
        xpath.url_xpath = '//div[@class="search_result"]//a[@class="result"]/@href'
        xpath.title_xpath = '//div[@class="search_result"]//a[@class="result"]'
        xpath.content_xpath = '//div[@class="search_result"]//p[@class="content"]'

        self.assertRaises(AttributeError, xpath.response, None)
        self.assertRaises(AttributeError, xpath.response, [])
        self.assertRaises(AttributeError, xpath.response, '')
        self.assertRaises(AttributeError, xpath.response, '[]')

        response = mock.Mock(text='<html></html>')
        self.assertEqual(xpath.response(response), [])

        html = u"""
        <div>
            <div class="search_result">
                <a class="result" href="https://result1.com">Result 1</a>
                <p class="content">Content 1</p>
                <a class="cached" href="https://cachedresult1.com">Cache</a>
            </div>
            <div class="search_result">
                <a class="result" href="https://result2.com">Result 2</a>
                <p class="content">Content 2</p>
                <a class="cached" href="https://cachedresult2.com">Cache</a>
            </div>
        </div>
        """
        response = mock.Mock(text=html)
        results = xpath.response(response)
        self.assertEqual(type(results), list)
        self.assertEqual(len(results), 2)
        self.assertEqual(results[0]['title'], 'Result 1')
        self.assertEqual(results[0]['url'], 'https://result1.com/')
        self.assertEqual(results[0]['content'], 'Content 1')
        self.assertEqual(results[1]['title'], 'Result 2')
        self.assertEqual(results[1]['url'], 'https://result2.com/')
        self.assertEqual(results[1]['content'], 'Content 2')

        # with cached urls, without results_xpath
        xpath.cached_xpath = '//div[@class="search_result"]//a[@class="cached"]/@href'
        results = xpath.response(response)
        self.assertEqual(type(results), list)
        self.assertEqual(len(results), 2)
        self.assertEqual(results[0]['cached_url'], 'https://cachedresult1.com')
        self.assertEqual(results[1]['cached_url'], 'https://cachedresult2.com')
        self.assertFalse(results[0].get('is_onion', False))

        # results are onion urls (no results_xpath)
        xpath.categories = ['onions']
        results = xpath.response(response)
        self.assertTrue(results[0]['is_onion'])

        # with results_xpath
        xpath.results_xpath = '//div[@class="search_result"]'
        xpath.url_xpath = './/a[@class="result"]/@href'
        xpath.title_xpath = './/a[@class="result"]'
        xpath.content_xpath = './/p[@class="content"]'
        xpath.cached_xpath = None
        xpath.categories = []

        self.assertRaises(AttributeError, xpath.response, None)
        self.assertRaises(AttributeError, xpath.response, [])
        self.assertRaises(AttributeError, xpath.response, '')
        self.assertRaises(AttributeError, xpath.response, '[]')

        response = mock.Mock(text='<html></html>')
        self.assertEqual(xpath.response(response), [])

        response = mock.Mock(text=html)
        results = xpath.response(response)
        self.assertEqual(type(results), list)
        self.assertEqual(len(results), 2)
        self.assertEqual(results[0]['title'], 'Result 1')
        self.assertEqual(results[0]['url'], 'https://result1.com/')
        self.assertEqual(results[0]['content'], 'Content 1')
        self.assertEqual(results[1]['title'], 'Result 2')
        self.assertEqual(results[1]['url'], 'https://result2.com/')
        self.assertEqual(results[1]['content'], 'Content 2')

        # with cached urls, with results_xpath
        xpath.cached_xpath = './/a[@class="cached"]/@href'
        results = xpath.response(response)
        self.assertEqual(type(results), list)
        self.assertEqual(len(results), 2)
        self.assertEqual(results[0]['cached_url'], 'https://cachedresult1.com')
        self.assertEqual(results[1]['cached_url'], 'https://cachedresult2.com')
        self.assertFalse(results[0].get('is_onion', False))

        # results are onion urls (with results_xpath)
        xpath.categories = ['onions']
        results = xpath.response(response)
        self.assertTrue(results[0]['is_onion'])