1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
|
# SPDX-License-Identifier: AGPL-3.0-or-later
"""Yahoo (News)
Yahoo News is "English only" and do not offer localized nor language queries.
"""
# pylint: disable=invalid-name, missing-function-docstring
import re
from urllib.parse import urlencode
from datetime import datetime, timedelta
from dateutil import parser
from lxml import html
from searx import logger
from searx.utils import (
eval_xpath_list,
eval_xpath_getindex,
extract_text,
)
from searx.engines.yahoo import parse_url
logger = logger.getChild('yahoo_news engine')
# about
about = {
"website": 'https://news.yahoo.com',
"wikidata_id": 'Q3044717',
"official_api_documentation": 'https://developer.yahoo.com/api/',
"use_official_api": False,
"require_api_key": False,
"results": 'HTML',
}
language_support = False
time_range_support = False
safesearch = False
paging = True
categories = ['news']
# search-url
search_url = (
'https://news.search.yahoo.com/search'
'?{query}&b={offset}'
)
AGO_RE = re.compile(r'([0-9]+)\s*(year|month|week|day|minute|hour)')
AGO_TIMEDELTA = {
'minute': timedelta(minutes=1),
'hour': timedelta(hours=1),
'day': timedelta(days=1),
'week': timedelta(days=7),
'month': timedelta(days=30),
'year': timedelta(days=365),
}
def request(query, params):
offset = (params['pageno'] - 1) * 10 + 1
params['url'] = search_url.format(
offset = offset,
query = urlencode({'p': query})
)
logger.debug("query_url --> %s", params['url'])
return params
def response(resp):
results = []
dom = html.fromstring(resp.text)
# parse results
for result in eval_xpath_list(dom, '//ol[contains(@class,"searchCenterMiddle")]//li'):
url = eval_xpath_getindex(result, './/h4/a/@href', 0, None)
if url is None:
continue
url = parse_url(url)
title = extract_text(result.xpath('.//h4/a'))
content = extract_text(result.xpath('.//p'))
img_src = eval_xpath_getindex(result, './/img/@data-src', 0, None)
item = {
'url': url,
'title': title,
'content': content,
'img_src' : img_src
}
pub_date = extract_text(result.xpath('.//span[contains(@class,"s-time")]'))
ago = AGO_RE.search(pub_date)
if ago:
number = int(ago.group(1))
delta = AGO_TIMEDELTA[ago.group(2)]
pub_date = datetime.now() - delta * number
else:
try:
pub_date = parser.parse(pub_date)
except parser.ParserError:
pub_date = None
if pub_date is not None:
item['publishedDate'] = pub_date
results.append(item)
for suggestion in eval_xpath_list(dom, '//div[contains(@class,"AlsoTry")]//td'):
results.append({'suggestion': extract_text(suggestion)})
return results
|
