[enh] add raise_for_httperror

check HTTP response:
* detect some comme CAPTCHA challenge (no solving). In this case the engine is suspended for long a time.
* otherwise raise HTTPError as before

the check is done in poolrequests.py (was before in search.py).

update qwant, wikipedia, wikidata to use raise_for_httperror instead of raise_for_status

1 files changed, 11 insertions, 0 deletions

diff --git a/searx/poolrequests.py b/searx/poolrequests.py
index 1eedc84b8..25a6baed9 100644
--- a/searx/poolrequests.py
+++ b/searx/poolrequests.py
@@ -7,6 +7,7 @@ import requests
 
 from searx import settings
 from searx import logger
+from searx.raise_for_httperror import raise_for_httperror
 
 
 logger = logger.getChild('poolrequests')
@@ -156,6 +157,12 @@ def request(method, url, **kwargs):
         if timeout is not None:
             kwargs['timeout'] = timeout
 
+    # raise_for_error
+    check_for_httperror = True
+    if 'raise_for_httperror' in kwargs:
+        check_for_httperror = kwargs['raise_for_httperror']
+        del kwargs['raise_for_httperror']
+
     # do request
     response = session.request(method=method, url=url, **kwargs)
 
@@ -176,6 +183,10 @@ def request(method, url, **kwargs):
     if hasattr(threadLocal, 'total_time'):
         threadLocal.total_time += time_after_request - time_before_request
 
+    # raise an exception
+    if check_for_httperror:
+        raise_for_httperror(response)
+
     return response