searx.network: add "verify" option to the networks

Each network can define a verify option:
* false to disable certificate verification
* a path to existing certificate.

SearXNG uses SSL_CERT_FILE and SSL_CERT_DIR when they are defined
see https://www.python-httpx.org/environment_variables/#ssl_cert_file

6 files changed, 26 insertions, 12 deletions

diff --git a/searx/network/client.py b/searx/network/client.py
index 11086dd33..f25aaf9ab 100644
--- a/searx/network/client.py
+++ b/searx/network/client.py
@@ -26,9 +26,6 @@ else:
 logger = logger.getChild('searx.network.client')
 LOOP = None
 SSLCONTEXTS: Dict[Any, SSLContext] = {}
-TRANSPORT_KWARGS = {
-    'trust_env': False,
-}
 
 
 def get_sslcontexts(proxy_url=None, cert=None, verify=True, trust_env=True, http2=False):
@@ -74,7 +71,7 @@ def get_transport_for_socks_proxy(verify, http2, local_address, proxy_url, limit
         rdns = True
 
     proxy_type, proxy_host, proxy_port, proxy_username, proxy_password = parse_proxy_url(proxy_url)
-    verify = get_sslcontexts(proxy_url, None, True, False, http2) if verify is True else verify
+    verify = get_sslcontexts(proxy_url, None, verify, True, http2) if verify is True else verify
     return AsyncProxyTransportFixed(
         proxy_type=proxy_type,
         proxy_host=proxy_host,
@@ -88,12 +85,11 @@ def get_transport_for_socks_proxy(verify, http2, local_address, proxy_url, limit
         local_address=local_address,
         limits=limit,
         retries=retries,
-        **TRANSPORT_KWARGS,
     )
 
 
 def get_transport(verify, http2, local_address, proxy_url, limit, retries):
-    verify = get_sslcontexts(None, None, True, False, http2) if verify is True else verify
+    verify = get_sslcontexts(None, None, verify, True, http2) if verify is True else verify
     return httpx.AsyncHTTPTransport(
         # pylint: disable=protected-access
         verify=verify,
@@ -102,7 +98,6 @@ def get_transport(verify, http2, local_address, proxy_url, limit, retries):
         proxy=httpx._config.Proxy(proxy_url) if proxy_url else None,
         local_address=local_address,
         retries=retries,
-        **TRANSPORT_KWARGS,
     )
 
 
diff --git a/searx/network/network.py b/searx/network/network.py
index 677a908bf..87c077f23 100644
--- a/searx/network/network.py
+++ b/searx/network/network.py
@@ -334,7 +334,7 @@ def initialize(settings_engines=None, settings_outgoing=None):
     # see https://github.com/encode/httpx/blob/e05a5372eb6172287458b37447c30f650047e1b8/httpx/_transports/default.py#L108-L121  # pylint: disable=line-too-long
     default_params = {
         'enable_http': False,
-        'verify': True,
+        'verify': settings_outgoing['verify'],
         'enable_http2': settings_outgoing['enable_http2'],
         'max_connections': settings_outgoing['pool_connections'],
         'max_keepalive_connections': settings_outgoing['pool_maxsize'],
diff --git a/searx/search/processors/online.py b/searx/search/processors/online.py
index be1ca57f3..dd5d1e36b 100644
--- a/searx/search/processors/online.py
+++ b/searx/search/processors/online.py
@@ -7,6 +7,7 @@
 
 from timeit import default_timer
 import asyncio
+import ssl
 import httpx
 
 import searx.network
@@ -29,7 +30,6 @@ def default_request_params():
         'data': {},
         'url': '',
         'cookies': {},
-        'verify': True,
         'auth': None
         # fmt: on
     }
@@ -76,9 +76,15 @@ class OnlineProcessor(EngineProcessor):
     def _send_http_request(self, params):
         # create dictionary which contain all
         # information about the request
-        request_args = dict(
-            headers=params['headers'], cookies=params['cookies'], verify=params['verify'], auth=params['auth']
-        )
+        request_args = dict(headers=params['headers'], cookies=params['cookies'], auth=params['auth'])
+
+        # verify
+        # if not None, it overrides the verify value defined in the network.
+        # use False to accept any server certificate
+        # use a path to file to specify a server certificate
+        verify = params.get('verify')
+        if verify is not None:
+            request_args['verify'] = params['verify']
 
         # max_redirects
         max_redirects = params.get('max_redirects')
@@ -153,6 +159,10 @@ class OnlineProcessor(EngineProcessor):
             # send requests and parse the results
             search_results = self._search_basic(query, params)
             self.extend_container(result_container, start_time, search_results)
+        except ssl.SSLError as e:
+            # requests timeout (connect or read)
+            self.handle_exception(result_container, e, suspend=True)
+            self.logger.error("SSLError {}, verify={}".format(e, searx.network.get_network(self.engine_name).verify))
         except (httpx.TimeoutException, asyncio.TimeoutError) as e:
             # requests timeout (connect or read)
             self.handle_exception(result_container, e, suspend=True)
diff --git a/searx/settings.yml b/searx/settings.yml
index f21d7f05a..2304c6fe7 100644
--- a/searx/settings.yml
+++ b/searx/settings.yml
@@ -145,6 +145,11 @@ outgoing:
   pool_maxsize: 20
   # See https://www.python-httpx.org/http2/
   enable_http2: true
+  # uncomment below section if you want to use a custom server certificate
+  # see https://www.python-httpx.org/advanced/#changing-the-verification-defaults
+  # and https://www.python-httpx.org/compatibility/#ssl-configuration
+  #  verify: ~/.mitmproxy/mitmproxy-ca-cert.cer
+  #
   # uncomment below section if you want to use a proxyq see: SOCKS proxies
   #   https://2.python-requests.org/en/latest/user/advanced/#proxies
   # are also supported: see
diff --git a/searx/settings_defaults.py b/searx/settings_defaults.py
index 330878c1f..6575b0b0c 100644
--- a/searx/settings_defaults.py
+++ b/searx/settings_defaults.py
@@ -199,6 +199,7 @@ SCHEMA = {
         'useragent_suffix': SettingsValue(str, ''),
         'request_timeout': SettingsValue(numbers.Real, 3.0),
         'enable_http2': SettingsValue(bool, True),
+        'verify': SettingsValue((bool, str), True),
         'max_request_timeout': SettingsValue((None, numbers.Real), None),
         # Magic number kept from previous code
         'pool_connections': SettingsValue(int, 100),
diff --git a/searx/webapp.py b/searx/webapp.py
index 688698bb6..5c3fbae8b 100755
--- a/searx/webapp.py
+++ b/searx/webapp.py
@@ -165,6 +165,7 @@ timeout_text = gettext('timeout')
 parsing_error_text = gettext('parsing error')
 http_protocol_error_text = gettext('HTTP protocol error')
 network_error_text = gettext('network error')
+ssl_cert_error_text = gettext("SSL error: certificate validation has failed")
 exception_classname_to_text = {
     None: gettext('unexpected crash'),
     'timeout': timeout_text,
@@ -189,6 +190,8 @@ exception_classname_to_text = {
     'KeyError': parsing_error_text,
     'json.decoder.JSONDecodeError': parsing_error_text,
     'lxml.etree.ParserError': parsing_error_text,
+    'ssl.SSLCertVerificationError': ssl_cert_error_text,  # for Python > 3.7
+    'ssl.CertificateError': ssl_cert_error_text,  # for Python 3.7
 }