1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
|
# SPDX-License-Identifier: AGPL-3.0-or-later
""".. sidebar:: info
- `builtwith.com Discourse <https://trends.builtwith.com/websitelist/Discourse>`_
Discourse is an open source Internet forum system. To search in a forum this
engine offers some additional settings:
- :py:obj:`base_url`
- :py:obj:`api_order`
- :py:obj:`search_endpoint`
- :py:obj:`show_avatar`
- :py:obj:`api_key`
- :py:obj:`api_username`
Example
=======
To search in your favorite Discourse forum, add a configuration like shown here
for the ``paddling.com`` forum:
.. code:: yaml
- name: paddling
engine: discourse
shortcut: paddle
base_url: 'https://forums.paddling.com/'
api_order: views
categories: ['social media', 'sports']
show_avatar: true
If the forum is private, you need to add an API key and username for the search:
.. code:: yaml
- name: paddling
engine: discourse
shortcut: paddle
base_url: 'https://forums.paddling.com/'
api_order: views
categories: ['social media', 'sports']
show_avatar: true
api_key: '<KEY>'
api_username: 'system'
Implementations
===============
"""
from urllib.parse import urlencode
from datetime import datetime, timedelta
import html
from dateutil import parser
from flask_babel import gettext
about = {
"website": "https://discourse.org/",
"wikidata_id": "Q15054354",
"official_api_documentation": "https://docs.discourse.org/",
"use_official_api": True,
"require_api_key": False,
"results": "JSON",
}
base_url: str = None # type: ignore
"""URL of the Discourse forum."""
search_endpoint = '/search.json'
"""URL path of the `search endpoint`_.
.. _search endpoint: https://docs.discourse.org/#tag/Search
"""
api_order = 'likes'
"""Order method, valid values are: ``latest``, ``likes``, ``views``, ``latest_topic``"""
show_avatar = False
"""Show avatar of the user who send the post."""
api_key = ''
"""API key of the Discourse forum."""
api_username = ''
"""API username of the Discourse forum."""
paging = True
time_range_support = True
AGO_TIMEDELTA = {
'day': timedelta(days=1),
'week': timedelta(days=7),
'month': timedelta(days=31),
'year': timedelta(days=365),
}
def request(query, params):
if len(query) <= 2:
return None
q = [query, f'order:{api_order}']
time_range = params.get('time_range')
if time_range:
after_date = datetime.now() - AGO_TIMEDELTA[time_range]
q.append('after:' + after_date.strftime('%Y-%m-%d'))
args = {
'q': ' '.join(q),
'page': params['pageno'],
}
params['url'] = f'{base_url}{search_endpoint}?{urlencode(args)}'
params['headers'] = {
'Accept': 'application/json, text/javascript, */*; q=0.01',
'X-Requested-With': 'XMLHttpRequest',
}
if api_key != '':
params['headers']['Api-Key'] = api_key
if api_username != '':
params['headers']['Api-Username'] = api_username
return params
def response(resp):
results = []
json_data = resp.json()
if ('topics' or 'posts') not in json_data.keys():
return []
topics = {}
for item in json_data['topics']:
topics[item['id']] = item
for post in json_data['posts']:
result = topics.get(post['topic_id'], {})
url = f"{base_url}/p/{post['id']}"
status = gettext("closed") if result.get('closed', '') else gettext("open")
comments = result.get('posts_count', 0)
publishedDate = parser.parse(result['created_at'])
metadata = []
metadata.append('@' + post.get('username', ''))
if int(comments) > 1:
metadata.append(f'{gettext("comments")}: {comments}')
if result.get('has_accepted_answer'):
metadata.append(gettext("answered"))
elif int(comments) > 1:
metadata.append(status)
result = {
'url': url,
'title': html.unescape(result['title']),
'content': html.unescape(post.get('blurb', '')),
'metadata': ' | '.join(metadata),
'publishedDate': publishedDate,
'upstream': {'topics': result},
}
avatar = post.get('avatar_template', '').replace('{size}', '96')
if show_avatar and avatar:
result['thumbnail'] = base_url + avatar
results.append(result)
results.append({'number_of_results': len(json_data['topics'])})
return results
|
