[enh] timeout and total HTTP time are managed by searx.poolrequests

1 files changed, 30 insertions, 40 deletions

diff --git a/searx/search.py b/searx/search.py
index 22aea3661..71d20724e 100644
--- a/searx/search.py
+++ b/searx/search.py
@@ -47,16 +47,12 @@ logger = logger.getChild('search')
 number_of_searches = 0
 
 
-def send_http_request(engine, request_params, start_time, timeout_limit):
-    # for page_load_time stats
-    time_before_request = time()
-
+def send_http_request(engine, request_params):
     # create dictionary which contain all
     # informations about the request
     request_args = dict(
         headers=request_params['headers'],
         cookies=request_params['cookies'],
-        timeout=timeout_limit,
         verify=request_params['verify']
     )
 
@@ -68,29 +64,10 @@ def send_http_request(engine, request_params, start_time, timeout_limit):
         request_args['data'] = request_params['data']
 
     # send the request
-    response = req(request_params['url'], **request_args)
-
-    # is there a timeout (no parsing in this case)
-    timeout_overhead = 0.2  # seconds
-    time_after_request = time()
-    search_duration = time_after_request - start_time
-    if search_duration > timeout_limit + timeout_overhead:
-        raise requests.exceptions.Timeout(response=response)
-
-    with threading.RLock():
-        # no error : reset the suspend variables
-        engine.continuous_errors = 0
-        engine.suspend_end_time = 0
-        # update stats with current page-load-time
-        # only the HTTP request
-        engine.stats['page_load_time'] += time_after_request - time_before_request
-        engine.stats['page_load_count'] += 1
+    return req(request_params['url'], **request_args)
 
-    # everything is ok : return the response
-    return response
 
-
-def search_one_request(engine, query, request_params, start_time, timeout_limit):
+def search_one_request(engine, query, request_params):
     # update request parameters dependent on
     # search-engine (contained in engines folder)
     engine.request(query, request_params)
@@ -103,7 +80,7 @@ def search_one_request(engine, query, request_params, start_time, timeout_limit)
         return []
 
     # send request
-    response = send_http_request(engine, request_params, start_time, timeout_limit)
+    response = send_http_request(engine, request_params)
 
     # parse the response
     response.search_params = request_params
@@ -111,11 +88,20 @@ def search_one_request(engine, query, request_params, start_time, timeout_limit)
 
 
 def search_one_request_safe(engine_name, query, request_params, result_container, start_time, timeout_limit):
+    # set timeout for all HTTP requests
+    requests_lib.set_timeout_for_thread(timeout_limit, start_time=start_time)
+    # reset the HTTP total time
+    requests_lib.reset_time_for_thread()
+
+    #
     engine = engines[engine_name]
 
+    # suppose everything will be alright
+    requests_exception = False
+
     try:
         # send requests and parse the results
-        search_results = search_one_request(engine, query, request_params, start_time, timeout_limit)
+        search_results = search_one_request(engine, query, request_params)
 
         # add results
         result_container.extend(engine_name, search_results)
@@ -124,14 +110,15 @@ def search_one_request_safe(engine_name, query, request_params, result_container
         with threading.RLock():
             engine.stats['engine_time'] += time() - start_time
             engine.stats['engine_time_count'] += 1
-
-        return True
+            # update stats with the total HTTP time
+            engine.stats['page_load_time'] += requests_lib.get_time_for_thread()
+            engine.stats['page_load_count'] += 1
 
     except Exception as e:
-        engine.stats['errors'] += 1
-
         search_duration = time() - start_time
-        requests_exception = False
+
+        with threading.RLock():
+            engine.stats['errors'] += 1
 
         if (issubclass(e.__class__, requests.exceptions.Timeout)):
             result_container.add_unresponsive_engine((engine_name, gettext('timeout')))
@@ -152,14 +139,17 @@ def search_one_request_safe(engine_name, query, request_params, result_container
             # others errors
             logger.exception('engine {0} : exception : {1}'.format(engine_name, e))
 
-        # update continuous_errors / suspend_end_time
+    # suspend or not the engine if there are HTTP errors
+    with threading.RLock():
         if requests_exception:
-            with threading.RLock():
-                engine.continuous_errors += 1
-                engine.suspend_end_time = time() + min(60, engine.continuous_errors)
-
-        #
-        return False
+            # update continuous_errors / suspend_end_time
+            engine.continuous_errors += 1
+            engine.suspend_end_time = time() + min(60, engine.continuous_errors)
+        else:
+            # no HTTP error (perhaps an engine error)
+            # anyway, reset the suspend variables
+            engine.continuous_errors = 0
+            engine.suspend_end_time = 0
 
 
 def search_multiple_requests(requests, result_container, start_time, timeout_limit):