diff options
Diffstat (limited to 'searx/poolrequests.py')
-rw-r--r-- | searx/poolrequests.py | 30 |
1 files changed, 28 insertions, 2 deletions
diff --git a/searx/poolrequests.py b/searx/poolrequests.py index e03797ce2..1eedc84b8 100644 --- a/searx/poolrequests.py +++ b/searx/poolrequests.py @@ -111,6 +111,32 @@ def get_time_for_thread(): return threadLocal.total_time +def get_proxy_cycles(proxy_settings): + if not proxy_settings: + return None + # Backwards compatibility for single proxy in settings.yml + for protocol, proxy in proxy_settings.items(): + if isinstance(proxy, str): + proxy_settings[protocol] = [proxy] + + for protocol in proxy_settings: + proxy_settings[protocol] = cycle(proxy_settings[protocol]) + return proxy_settings + + +GLOBAL_PROXY_CYCLES = get_proxy_cycles(settings['outgoing'].get('proxies')) + + +def get_proxies(proxy_cycles): + if proxy_cycles: + return {protocol: next(proxy_cycle) for protocol, proxy_cycle in proxy_cycles.items()} + return None + + +def get_global_proxies(): + return get_proxies(GLOBAL_PROXY_CYCLES) + + def request(method, url, **kwargs): """same as requests/requests/api.py request(...)""" time_before_request = time() @@ -119,8 +145,8 @@ def request(method, url, **kwargs): session = SessionSinglePool() # proxies - if kwargs.get('proxies') is None: - kwargs['proxies'] = settings['outgoing'].get('proxies') + if not kwargs.get('proxies'): + kwargs['proxies'] = get_global_proxies() # timeout if 'timeout' in kwargs: |