diff options
author | volth <volth@webmaster.ms> | 2019-07-25 06:40:48 +0000 |
---|---|---|
committer | Alexandre Flament <alex@al-f.net> | 2019-07-25 08:40:48 +0200 |
commit | eb182df1324f84d6864f324551fe6be7c535e451 (patch) | |
tree | 90df3f071b6c8ee7ecbd3d6d8a6e201b3583ca47 | |
parent | 3b1122c5fa1bbac49f6e0eb9ff69efb8dca996ea (diff) | |
download | searxng-eb182df1324f84d6864f324551fe6be7c535e451.tar.gz searxng-eb182df1324f84d6864f324551fe6be7c535e451.zip |
[mod] restore btdigg engine as btdig.com (#1515)
-rw-r--r-- | searx/engines/btdigg.py | 39 | ||||
-rw-r--r-- | searx/settings.yml | 4 | ||||
-rw-r--r-- | searx/templates/courgette/result_templates/torrent.html | 2 | ||||
-rw-r--r-- | searx/templates/legacy/result_templates/torrent.html | 2 | ||||
-rw-r--r-- | searx/templates/oscar/result_templates/torrent.html | 2 | ||||
-rw-r--r-- | tests/unit/engines/test_btdigg.py | 414 |
6 files changed, 94 insertions, 369 deletions
diff --git a/searx/engines/btdigg.py b/searx/engines/btdigg.py index 40438673f..82eedc24b 100644 --- a/searx/engines/btdigg.py +++ b/searx/engines/btdigg.py @@ -1,7 +1,7 @@ """ BTDigg (Videos, Music, Files) - @website https://btdigg.org + @website https://btdig.com @provide-api yes (on demand) @using-api no @@ -21,7 +21,7 @@ categories = ['videos', 'music', 'files'] paging = True # search-url -url = 'https://btdigg.org' +url = 'https://btdig.com' search_url = url + '/search?q={search_term}&p={pageno}' @@ -39,7 +39,7 @@ def response(resp): dom = html.fromstring(resp.text) - search_res = dom.xpath('//div[@id="search_res"]/table/tr') + search_res = dom.xpath('//div[@class="one_result"]') # return empty array if nothing is found if not search_res: @@ -47,46 +47,39 @@ def response(resp): # parse results for result in search_res: - link = result.xpath('.//td[@class="torrent_name"]//a')[0] + link = result.xpath('.//div[@class="torrent_name"]//a')[0] href = urljoin(url, link.attrib.get('href')) title = extract_text(link) - content = extract_text(result.xpath('.//pre[@class="snippet"]')[0]) - content = "<br />".join(content.split("\n")) - filesize = result.xpath('.//span[@class="attr_val"]/text()')[0].split()[0] - filesize_multiplier = result.xpath('.//span[@class="attr_val"]/text()')[0].split()[1] - files = result.xpath('.//span[@class="attr_val"]/text()')[1] - seed = result.xpath('.//span[@class="attr_val"]/text()')[2] + excerpt = result.xpath('.//div[@class="torrent_excerpt"]')[0] + content = html.tostring(excerpt, encoding='unicode', method='text', with_tail=False) + # it is better to emit <br/> instead of |, but html tags are verboten + content = content.strip().replace('\n', ' | ') + content = ' '.join(content.split()) - # convert seed to int if possible - if seed.isdigit(): - seed = int(seed) - else: - seed = 0 - - leech = 0 + filesize = result.xpath('.//span[@class="torrent_size"]/text()')[0].split()[0] + filesize_multiplier = result.xpath('.//span[@class="torrent_size"]/text()')[0].split()[1] + files = (result.xpath('.//span[@class="torrent_files"]/text()') or ['1'])[0] # convert filesize to byte if possible filesize = get_torrent_size(filesize, filesize_multiplier) # convert files to int if possible - if files.isdigit(): + try: files = int(files) - else: + except: files = None - magnetlink = result.xpath('.//td[@class="ttth"]//a')[0].attrib['href'] + magnetlink = result.xpath('.//div[@class="torrent_magnet"]//a')[0].attrib['href'] # append result results.append({'url': href, 'title': title, 'content': content, - 'seed': seed, - 'leech': leech, 'filesize': filesize, 'files': files, 'magnetlink': magnetlink, 'template': 'torrent.html'}) # return results sorted by seeder - return sorted(results, key=itemgetter('seed'), reverse=True) + return results diff --git a/searx/settings.yml b/searx/settings.yml index 53dfaae2c..c2a77630a 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -115,6 +115,10 @@ engines: disabled : True shortcut : bb + - name : btdigg + engine : btdigg + shortcut : bt + - name : ccc-tv engine : xpath paging : False diff --git a/searx/templates/courgette/result_templates/torrent.html b/searx/templates/courgette/result_templates/torrent.html index 2fd8395ad..d659064d9 100644 --- a/searx/templates/courgette/result_templates/torrent.html +++ b/searx/templates/courgette/result_templates/torrent.html @@ -4,7 +4,7 @@ {% endif %} <h3 class="result_title"><a href="{{ result.url }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %}>{{ result.title|safe }}</a></h3> {% if result.content %}<span class="content">{{ result.content|safe }}</span><br />{% endif %} - <span class="stats">{{ _('Seeder') }} : {{ result.seed }}, {{ _('Leecher') }} : {{ result.leech }}</span><br /> + {% if result.seed %}<span class="stats">{{ _('Seeder') }} : {{ result.seed }}, {{ _('Leecher') }} : {{ result.leech }}</span><br />{% endif %} <span> {% if result.magnetlink %}<a href="{{ result.magnetlink }}" class="magnetlink">{{ _('magnet link') }}</a>{% endif %} {% if result.torrentfile %}<a href="{{ result.torrentfile }}" class="torrentfile" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %}>{{ _('torrent file') }}</a>{% endif %} diff --git a/searx/templates/legacy/result_templates/torrent.html b/searx/templates/legacy/result_templates/torrent.html index 67e058ae5..7a8ac33de 100644 --- a/searx/templates/legacy/result_templates/torrent.html +++ b/searx/templates/legacy/result_templates/torrent.html @@ -8,6 +8,6 @@ <p> {% if result.magnetlink %}<a href="{{ result.magnetlink }}" class="magnetlink">{{ _('magnet link') }}</a>{% endif %} {% if result.torrentfile %}<a href="{{ result.torrentfile }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %} class="torrentfile">{{ _('torrent file') }}</a>{% endif %} - - <span class="stats">{{ _('Seeder') }} : {{ result.seed }}, {{ _('Leecher') }} : {{ result.leech }}</span> + {% if result.seed %}<span class="stats">{{ _('Seeder') }} : {{ result.seed }}, {{ _('Leecher') }} : {{ result.leech }}</span>{% endif %} </p> </div> diff --git a/searx/templates/oscar/result_templates/torrent.html b/searx/templates/oscar/result_templates/torrent.html index bc2b30fbe..f5ea415e2 100644 --- a/searx/templates/oscar/result_templates/torrent.html +++ b/searx/templates/oscar/result_templates/torrent.html @@ -3,7 +3,7 @@ {{ result_header(result, favicons) }} {{ result_sub_header(result) }} -<p class="result-content">{{ icon('transfer') }} {{ _('Seeder') }} <span class="badge">{{ result.seed }}</span> • {{ _('Leecher') }} <span class="badge">{{ result.leech }}</span> +{% if result.seed %}<p class="result-content">{{ icon('transfer') }} {{ _('Seeder') }} <span class="badge">{{ result.seed }}</span> • {{ _('Leecher') }} <span class="badge">{{ result.leech }}</span>{% endif %} {% if result.filesize %}<br />{{ icon('floppy-disk') }} {{ _('Filesize') }} <span class="badge"> {% if result.filesize < 1024 %}{{ result.filesize }} {{ _('Bytes') }} diff --git a/tests/unit/engines/test_btdigg.py b/tests/unit/engines/test_btdigg.py index 6a88e3f75..45ddaa6e3 100644 --- a/tests/unit/engines/test_btdigg.py +++ b/tests/unit/engines/test_btdigg.py @@ -14,7 +14,7 @@ class TestBtdiggEngine(SearxTestCase): params = btdigg.request(query, dicto) self.assertIn('url', params) self.assertIn(query, params['url']) - self.assertIn('btdigg.org', params['url']) + self.assertIn('btdig.com', params['url']) def test_response(self): self.assertRaises(AttributeError, btdigg.response, None) @@ -26,359 +26,87 @@ class TestBtdiggEngine(SearxTestCase): self.assertEqual(btdigg.response(response), []) html = u""" - <div id="search_res"> - <table> - <tr> - <td class="idx">1</td> - <td> - <table class="torrent_name_tbl"> - <tr> - <td class="torrent_name"> - <a href="/url">Should be the title</a> - </td> - </tr> - </table> - <table class="torrent_name_tbl"> - <tr> - <td class="ttth"> - <a onclick="fclck(this.href)" href="magnet:?xt=urn:btih:magnet&dn=Test" - title="Télécharger des liens Magnet">[magnet]</a> - </td> - <td class="ttth"> - <a href="https://btcloud.io/manager?cmd=add&info_hash=hash" - target="_blank" title="Ajouter à BTCloud">[cloud]</a> - </td> - <td> - <span class="attr_name">Taille:</span> - <span class="attr_val">8 B</span> - </td> - <td> - <span class="attr_name">Fichiers:</span> - <span class="attr_val">710</span> - </td> - <td> - <span class="attr_name">Téléchargements:</span> - <span class="attr_val">5</span> - </td> - <td> - <span class="attr_name">Temps:</span> - <span class="attr_val">417.8 jours</span> - </td> - <td> - <span class="attr_name">Dernière mise à jour:</span> - <span class="attr_val">5.3 jours</span> - </td> - <td> - <span class="attr_name">Faux:</span> - <span class="attr_val">Aucun</span> - </td> - </tr> - </table> - <pre class="snippet"> - Content - </pre> - </td> - </tr> - </table> + <div class="one_result" style="display:table-row;background-color:#e8e8e8"> + <div style="display:table-cell;color:rgb(0, 0, 0)"> + <div style="display:table"> + <div style="display:table-row"> + <div class="torrent_name" style="display:table-cell"> + <a style="color:rgb(0, 0, 204);text-decoration:underline;font-size:150%" + href="http://btdig.com/a72f35b7ee3a10928f02bb799e40ae5db701ed1c/pdf?q=pdf&p=1&order=0" + >3.9GBdeLibrosByHuasoFromHell(3de4)</a> + </div> + </div> + </div> + <div style="display:table"> + <div style="display:table-row"> + <div style="display:table-cell"> + <span class="torrent_files" style="color:#666;padding-left:10px">4217</span> files <span + class="torrent_size" style="color:#666;padding-left:10px">1 GB</span><span + class="torrent_age" style="color:rgb(0, 102, 0);padding-left:10px;margin: 0px 4px" + >found 3 years ago</span> + </div> + </div> + </div> + <div style="display:table;width:100%;padding:10px"> + <div style="display:table-row"> + <div class="torrent_magnet" style="display:table-cell"> + <div class="fa fa-magnet" style="color:#cc0000"> + <a href="magnet:?xt=urn:btih:a72f35b7ee3a10928f02bb799e40ae5db701ed1c&dn=3.9GBdeLibrosBy..." + title="Download via magnet-link"> magnet:?xt=urn:btih:a72f35b7ee...</a> + </div> + </div> + <div style="display:table-cell;color:rgb(0, 0, 0);text-align:right"> + <span style="color:rgb(136, 136, 136);margin: 0px 0px 0px 4px"></span><span + style="color:rgb(0, 102, 0);margin: 0px 4px">found 3 years ago</span> + </div> + </div> + </div> + <div class="torrent_excerpt" style="display:table;padding:10px;white-space:nowrap"> + <div class="fa fa-folder-open" style="padding-left:0em"> 3.9GBdeLibrosByHuasoFromHell(3de4)</div><br/> + <div class="fa fa-folder-open" style="padding-left:1em"> Libros H-Z</div><br/> + <div class="fa fa-folder-open" style="padding-left:2em"> H</div><br/><div class="fa fa-file-archive-o" + style="padding-left:3em"> H.H. Hollis - El truco de la espada-<b + style="color:red; background-color:yellow">pdf</b>.zip</div><span + style="color:#666;padding-left:10px">17 KB</span><br/> + <div class="fa fa-file-archive-o" style="padding-left:3em"> Hagakure - El Libro del Samurai-<b + style="color:red; background-color:yellow">pdf</b>.zip</div><span + style="color:#666;padding-left:10px">95 KB</span><br/> + <div class="fa fa-folder-open" style="padding-left:3em"> Hamsun, Knut (1859-1952)</div><br/> + <div class="fa fa-file-archive-o" style="padding-left:4em"> Hamsun, Knut - Hambre-<b + style="color:red; background-color:yellow">pdf</b>.zip</div><span + style="color:#666;padding-left:10px">786 KB</span><br/> + <div class="fa fa-plus-circle"><a + href="http://btdig.com/a72f35b7ee3a10928f02bb799e40ae5db701ed1c/pdf?q=pdf&p=1&order=0" + > 4214 hidden files<span style="color:#666;padding-left:10px">1 GB</span></a></div> + </div> + </div> </div> """ response = mock.Mock(text=html.encode('utf-8')) results = btdigg.response(response) self.assertEqual(type(results), list) self.assertEqual(len(results), 1) - self.assertEqual(results[0]['title'], 'Should be the title') - self.assertEqual(results[0]['url'], 'https://btdigg.org/url') - self.assertEqual(results[0]['content'], 'Content') - self.assertEqual(results[0]['seed'], 5) - self.assertEqual(results[0]['leech'], 0) - self.assertEqual(results[0]['filesize'], 8) - self.assertEqual(results[0]['files'], 710) - self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:magnet&dn=Test') + self.assertEqual(results[0]['title'], '3.9GBdeLibrosByHuasoFromHell(3de4)') + self.assertEqual(results[0]['url'], + 'http://btdig.com/a72f35b7ee3a10928f02bb799e40ae5db701ed1c/pdf?q=pdf&p=1&order=0') + self.assertEqual(results[0]['content'], + '3.9GBdeLibrosByHuasoFromHell(3de4) | ' + + 'Libros H-Z | ' + + 'H H.H. Hollis - El truco de la espada-pdf.zip17 KB | ' + + 'Hagakure - El Libro del Samurai-pdf.zip95 KB | ' + + 'Hamsun, Knut (1859-1952) | Hamsun, Knut - Hambre-pdf.zip786 KB | ' + + '4214 hidden files1 GB') + self.assertEqual(results[0]['filesize'], 1 * 1024 * 1024 * 1024) + self.assertEqual(results[0]['files'], 4217) + self.assertEqual(results[0]['magnetlink'], + 'magnet:?xt=urn:btih:a72f35b7ee3a10928f02bb799e40ae5db701ed1c&dn=3.9GBdeLibrosBy...') html = """ - <div id="search_res"> - <table> - </table> - </div> - """ - response = mock.Mock(text=html.encode('utf-8')) - results = btdigg.response(response) - self.assertEqual(type(results), list) - self.assertEqual(len(results), 0) + <div style="display:table-row;background-color:#e8e8e8"> - html = u""" - <div id="search_res"> - <table> - <tr> - <td class="idx">1</td> - <td> - <table class="torrent_name_tbl"> - <tr> - <td class="torrent_name"> - <a href="/url">Should be the title</a> - </td> - </tr> - </table> - <table class="torrent_name_tbl"> - <tr> - <td class="ttth"> - <a onclick="fclck(this.href)" href="magnet:?xt=urn:btih:magnet&dn=Test" - title="Télécharger des liens Magnet">[magnet]</a> - </td> - <td class="ttth"> - <a href="https://btcloud.io/manager?cmd=add&info_hash=hash" - target="_blank" title="Ajouter à BTCloud">[cloud]</a> - </td> - <td> - <span class="attr_name">Taille:</span> - <span class="attr_val">1 KB</span> - </td> - <td> - <span class="attr_name">Fichiers:</span> - <span class="attr_val">710</span> - </td> - <td> - <span class="attr_name">Téléchargements:</span> - <span class="attr_val">5</span> - </td> - <td> - <span class="attr_name">Temps:</span> - <span class="attr_val">417.8 jours</span> - </td> - <td> - <span class="attr_name">Dernière mise à jour:</span> - <span class="attr_val">5.3 jours</span> - </td> - <td> - <span class="attr_name">Faux:</span> - <span class="attr_val">Aucun</span> - </td> - </tr> - </table> - <pre class="snippet"> - Content - </pre> - </td> - </tr> - <tr> - <td class="idx">1</td> - <td> - <table class="torrent_name_tbl"> - <tr> - <td class="torrent_name"> - <a href="/url">Should be the title</a> - </td> - </tr> - </table> - <table class="torrent_name_tbl"> - <tr> - <td class="ttth"> - <a onclick="fclck(this.href)" href="magnet:?xt=urn:btih:magnet&dn=Test" - title="Télécharger des liens Magnet">[magnet]</a> - </td> - <td class="ttth"> - <a href="https://btcloud.io/manager?cmd=add&info_hash=hash" - target="_blank" title="Ajouter à BTCloud">[cloud]</a> - </td> - <td> - <span class="attr_name">Taille:</span> - <span class="attr_val">1 MB</span> - </td> - <td> - <span class="attr_name">Fichiers:</span> - <span class="attr_val">a</span> - </td> - <td> - <span class="attr_name">Téléchargements:</span> - <span class="attr_val">4</span> - </td> - <td> - <span class="attr_name">Temps:</span> - <span class="attr_val">417.8 jours</span> - </td> - <td> - <span class="attr_name">Dernière mise à jour:</span> - <span class="attr_val">5.3 jours</span> - </td> - <td> - <span class="attr_name">Faux:</span> - <span class="attr_val">Aucun</span> - </td> - </tr> - </table> - <pre class="snippet"> - Content - </pre> - </td> - </tr> - <tr> - <td class="idx">1</td> - <td> - <table class="torrent_name_tbl"> - <tr> - <td class="torrent_name"> - <a href="/url">Should be the title</a> - </td> - </tr> - </table> - <table class="torrent_name_tbl"> - <tr> - <td class="ttth"> - <a onclick="fclck(this.href)" href="magnet:?xt=urn:btih:magnet&dn=Test" - title="Télécharger des liens Magnet">[magnet]</a> - </td> - <td class="ttth"> - <a href="https://btcloud.io/manager?cmd=add&info_hash=hash" - target="_blank" title="Ajouter à BTCloud">[cloud]</a> - </td> - <td> - <span class="attr_name">Taille:</span> - <span class="attr_val">1 GB</span> - </td> - <td> - <span class="attr_name">Fichiers:</span> - <span class="attr_val">710</span> - </td> - <td> - <span class="attr_name">Téléchargements:</span> - <span class="attr_val">3</span> - </td> - <td> - <span class="attr_name">Temps:</span> - <span class="attr_val">417.8 jours</span> - </td> - <td> - <span class="attr_name">Dernière mise à jour:</span> - <span class="attr_val">5.3 jours</span> - </td> - <td> - <span class="attr_name">Faux:</span> - <span class="attr_val">Aucun</span> - </td> - </tr> - </table> - <pre class="snippet"> - Content - </pre> - </td> - </tr> - <tr> - <td class="idx">1</td> - <td> - <table class="torrent_name_tbl"> - <tr> - <td class="torrent_name"> - <a href="/url">Should be the title</a> - </td> - </tr> - </table> - <table class="torrent_name_tbl"> - <tr> - <td class="ttth"> - <a onclick="fclck(this.href)" href="magnet:?xt=urn:btih:magnet&dn=Test" - title="Télécharger des liens Magnet">[magnet]</a> - </td> - <td class="ttth"> - <a href="https://btcloud.io/manager?cmd=add&info_hash=hash" - target="_blank" title="Ajouter à BTCloud">[cloud]</a> - </td> - <td> - <span class="attr_name">Taille:</span> - <span class="attr_val">1 TB</span> - </td> - <td> - <span class="attr_name">Fichiers:</span> - <span class="attr_val">710</span> - </td> - <td> - <span class="attr_name">Téléchargements:</span> - <span class="attr_val">2</span> - </td> - <td> - <span class="attr_name">Temps:</span> - <span class="attr_val">417.8 jours</span> - </td> - <td> - <span class="attr_name">Dernière mise à jour:</span> - <span class="attr_val">5.3 jours</span> - </td> - <td> - <span class="attr_name">Faux:</span> - <span class="attr_val">Aucun</span> - </td> - </tr> - </table> - <pre class="snippet"> - Content - </pre> - </td> - </tr> - <tr> - <td class="idx">1</td> - <td> - <table class="torrent_name_tbl"> - <tr> - <td class="torrent_name"> - <a href="/url">Should be the title</a> - </td> - </tr> - </table> - <table class="torrent_name_tbl"> - <tr> - <td class="ttth"> - <a onclick="fclck(this.href)" href="magnet:?xt=urn:btih:magnet&dn=Test" - title="Télécharger des liens Magnet">[magnet]</a> - </td> - <td class="ttth"> - <a href="https://btcloud.io/manager?cmd=add&info_hash=hash" - target="_blank" title="Ajouter à BTCloud">[cloud]</a> - </td> - <td> - <span class="attr_name">Taille:</span> - <span class="attr_val">a TB</span> - </td> - <td> - <span class="attr_name">Fichiers:</span> - <span class="attr_val">710</span> - </td> - <td> - <span class="attr_name">Téléchargements:</span> - <span class="attr_val">z</span> - </td> - <td> - <span class="attr_name">Temps:</span> - <span class="attr_val">417.8 jours</span> - </td> - <td> - <span class="attr_name">Dernière mise à jour:</span> - <span class="attr_val">5.3 jours</span> - </td> - <td> - <span class="attr_name">Faux:</span> - <span class="attr_val">Aucun</span> - </td> - </tr> - </table> - <pre class="snippet"> - Content - </pre> - </td> - </tr> - </table> </div> """ response = mock.Mock(text=html.encode('utf-8')) results = btdigg.response(response) self.assertEqual(type(results), list) - self.assertEqual(len(results), 5) - self.assertEqual(results[0]['title'], 'Should be the title') - self.assertEqual(results[0]['url'], 'https://btdigg.org/url') - self.assertEqual(results[0]['content'], 'Content') - self.assertEqual(results[0]['seed'], 5) - self.assertEqual(results[0]['leech'], 0) - self.assertEqual(results[0]['files'], 710) - self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:magnet&dn=Test') - self.assertEqual(results[0]['filesize'], 1024) - self.assertEqual(results[1]['filesize'], 1048576) - self.assertEqual(results[2]['filesize'], 1073741824) - self.assertEqual(results[3]['filesize'], 1099511627776) + self.assertEqual(len(results), 0) |