[httpx] replace searx.poolrequests by searx.network

settings.yml:

* outgoing.networks:
   * can contains network definition
   * propertiers: enable_http, verify, http2, max_connections, max_keepalive_connections,
     keepalive_expiry, local_addresses, support_ipv4, support_ipv6, proxies, max_redirects, retries
   * retries: 0 by default, number of times searx retries to send the HTTP request (using different IP & proxy each time)
   * local_addresses can be "192.168.0.1/24" (it supports IPv6)
   * support_ipv4 & support_ipv6: both True by default
     see https://github.com/searx/searx/pull/1034
* each engine can define a "network" section:
   * either a full network description
   * either reference an existing network

* all HTTP requests of engine use the same HTTP configuration (it was not the case before, see proxy configuration in master)

14 files changed, 18 insertions, 17 deletions

diff --git a/searx/engines/__init__.py b/searx/engines/__init__.py
index 2238ea1b9..95eda6dde 100644
--- a/searx/engines/__init__.py
+++ b/searx/engines/__init__.py
@@ -27,7 +27,7 @@ from searx import settings
 from searx import logger
 from searx.data import ENGINES_LANGUAGES
 from searx.exceptions import SearxEngineResponseException
-from searx.poolrequests import get, get_proxy_cycles
+from searx.network import get, initialize as initialize_network, set_context_network_name
 from searx.utils import load_module, match_language, get_engine_from_settings, gen_useragent
 
 
@@ -89,8 +89,6 @@ def load_engine(engine_data):
                 engine.categories = []
             else:
                 engine.categories = list(map(str.strip, param_value.split(',')))
-        elif param_name == 'proxies':
-            engine.proxies = get_proxy_cycles(param_value)
         else:
             setattr(engine, param_name, param_value)
 
@@ -289,9 +287,11 @@ def load_engines(engine_list):
 
 def initialize_engines(engine_list):
     load_engines(engine_list)
+    initialize_network(engine_list, settings['outgoing'])
 
     def engine_init(engine_name, init_fn):
         try:
+            set_context_network_name(engine_name)
             init_fn(get_engine_from_settings(engine_name))
         except SearxEngineResponseException as exc:
             logger.warn('%s engine: Fail to initialize // %s', engine_name, exc)
diff --git a/searx/engines/duckduckgo.py b/searx/engines/duckduckgo.py
index ae1e36686..3c086f81b 100644
--- a/searx/engines/duckduckgo.py
+++ b/searx/engines/duckduckgo.py
@@ -6,7 +6,7 @@
 from lxml.html import fromstring
 from json import loads
 from searx.utils import extract_text, match_language, eval_xpath, dict_subset
-from searx.poolrequests import get
+from searx.network import get
 
 # about
 about = {
diff --git a/searx/engines/duckduckgo_images.py b/searx/engines/duckduckgo_images.py
index 305eb1ca1..0daaf41e9 100644
--- a/searx/engines/duckduckgo_images.py
+++ b/searx/engines/duckduckgo_images.py
@@ -8,7 +8,7 @@ from urllib.parse import urlencode
 from searx.exceptions import SearxEngineAPIException
 from searx.engines.duckduckgo import get_region_code
 from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url  # NOQA # pylint: disable=unused-import
-from searx.poolrequests import get
+from searx.network import get
 
 # about
 about = {
diff --git a/searx/engines/gigablast.py b/searx/engines/gigablast.py
index 248991df9..bbd9e20d2 100644
--- a/searx/engines/gigablast.py
+++ b/searx/engines/gigablast.py
@@ -8,7 +8,7 @@ import re
 from json import loads
 from urllib.parse import urlencode
 # from searx import logger
-from searx.poolrequests import get
+from searx.network import get
 
 # about
 about = {
diff --git a/searx/engines/pubmed.py b/searx/engines/pubmed.py
index da02f91ca..5d88d398e 100644
--- a/searx/engines/pubmed.py
+++ b/searx/engines/pubmed.py
@@ -7,7 +7,7 @@ from flask_babel import gettext
 from lxml import etree
 from datetime import datetime
 from urllib.parse import urlencode
-from searx.poolrequests import get
+from searx.network import get
 
 # about
 about = {
diff --git a/searx/engines/qwant.py b/searx/engines/qwant.py
index 13dcf1250..d01dc0acc 100644
--- a/searx/engines/qwant.py
+++ b/searx/engines/qwant.py
@@ -8,7 +8,7 @@ from json import loads
 from urllib.parse import urlencode
 from searx.utils import html_to_text, match_language
 from searx.exceptions import SearxEngineAPIException, SearxEngineCaptchaException
-from searx.raise_for_httperror import raise_for_httperror
+from searx.network import raise_for_httperror
 
 # about
 about = {
diff --git a/searx/engines/seznam.py b/searx/engines/seznam.py
index c058ebb76..042088dbe 100644
--- a/searx/engines/seznam.py
+++ b/searx/engines/seznam.py
@@ -5,7 +5,7 @@
 
 from urllib.parse import urlencode
 from lxml import html
-from searx.poolrequests import get
+from searx.network import get
 from searx.exceptions import SearxEngineAccessDeniedException
 from searx.utils import (
     extract_text,
diff --git a/searx/engines/soundcloud.py b/searx/engines/soundcloud.py
index b3e3383bd..a6f923855 100644
--- a/searx/engines/soundcloud.py
+++ b/searx/engines/soundcloud.py
@@ -9,7 +9,7 @@ from lxml import html
 from dateutil import parser
 from urllib.parse import quote_plus, urlencode
 from searx import logger
-from searx.poolrequests import get as http_get
+from searx.network import get as http_get
 
 # about
 about = {
diff --git a/searx/engines/spotify.py b/searx/engines/spotify.py
index fbabec0bb..6816fe672 100644
--- a/searx/engines/spotify.py
+++ b/searx/engines/spotify.py
@@ -5,9 +5,10 @@
 
 from json import loads
 from urllib.parse import urlencode
-import searx.poolrequests as requests
 import base64
 
+from searx.network import post as http_post
+
 # about
 about = {
     "website": 'https://www.spotify.com',
@@ -38,7 +39,7 @@ def request(query, params):
 
     params['url'] = search_url.format(query=urlencode({'q': query}), offset=offset)
 
-    r = requests.post(
+    r = http_post(
         'https://accounts.spotify.com/api/token',
         data={'grant_type': 'client_credentials'},
         headers={'Authorization': 'Basic ' + base64.b64encode(
diff --git a/searx/engines/wikidata.py b/searx/engines/wikidata.py
index c8e4cfae6..ddcce9085 100644
--- a/searx/engines/wikidata.py
+++ b/searx/engines/wikidata.py
@@ -12,7 +12,7 @@ from babel.dates import format_datetime, format_date, format_time, get_datetime_
 
 from searx import logger
 from searx.data import WIKIDATA_UNITS
-from searx.poolrequests import post, get
+from searx.network import post, get
 from searx.utils import match_language, searx_useragent, get_string_replaces_function
 from searx.external_urls import get_external_url, get_earth_coordinates_url, area_to_osm_zoom
 from searx.engines.wikipedia import _fetch_supported_languages, supported_languages_url  # NOQA # pylint: disable=unused-import
diff --git a/searx/engines/wikipedia.py b/searx/engines/wikipedia.py
index 3ad8748fb..5e34db9a7 100644
--- a/searx/engines/wikipedia.py
+++ b/searx/engines/wikipedia.py
@@ -7,7 +7,7 @@ from urllib.parse import quote
 from json import loads
 from lxml.html import fromstring
 from searx.utils import match_language, searx_useragent
-from searx.raise_for_httperror import raise_for_httperror
+from searx.network import raise_for_httperror
 
 # about
 about = {
diff --git a/searx/engines/wolframalpha_noapi.py b/searx/engines/wolframalpha_noapi.py
index 8e427d575..1f2cfa4e6 100644
--- a/searx/engines/wolframalpha_noapi.py
+++ b/searx/engines/wolframalpha_noapi.py
@@ -7,7 +7,7 @@ from json import loads
 from time import time
 from urllib.parse import urlencode
 
-from searx.poolrequests import get as http_get
+from searx.network import get as http_get
 
 # about
 about = {
diff --git a/searx/engines/wordnik.py b/searx/engines/wordnik.py
index 3abe9efa2..4bfeb4070 100644
--- a/searx/engines/wordnik.py
+++ b/searx/engines/wordnik.py
@@ -6,7 +6,7 @@
 from lxml.html import fromstring
 from searx import logger
 from searx.utils import extract_text
-from searx.raise_for_httperror import raise_for_httperror
+from searx.network import raise_for_httperror
 
 logger = logger.getChild('Wordnik engine')
 
diff --git a/searx/engines/yggtorrent.py b/searx/engines/yggtorrent.py
index 8dfc0a0f2..f5af91f46 100644
--- a/searx/engines/yggtorrent.py
+++ b/searx/engines/yggtorrent.py
@@ -8,7 +8,7 @@ from operator import itemgetter
 from datetime import datetime
 from urllib.parse import quote
 from searx.utils import extract_text, get_torrent_size
-from searx.poolrequests import get as http_get
+from searx.network import get as http_get
 
 # about
 about = {