diff options
| -rw-r--r-- | dockerfiles/uwsgi.ini | 3 | ||||
| -rw-r--r-- | searx/shared/__init__.py | 31 | ||||
| -rw-r--r-- | searx/shared/shared_abstract.py | 15 | ||||
| -rw-r--r-- | searx/shared/shared_simple.py | 38 | ||||
| -rw-r--r-- | searx/shared/shared_uwsgi.py | 62 | ||||
| -rw-r--r-- | utils/templates/etc/uwsgi/apps-archlinux/searx.ini | 5 | ||||
| -rw-r--r-- | utils/templates/etc/uwsgi/apps-available/searx.ini | 3 |
7 files changed, 156 insertions, 1 deletions
diff --git a/dockerfiles/uwsgi.ini b/dockerfiles/uwsgi.ini index 398a440d9..818a99cc0 100644 --- a/dockerfiles/uwsgi.ini +++ b/dockerfiles/uwsgi.ini @@ -42,3 +42,6 @@ static-map = /static=/usr/local/searx/searx/static static-expires = /* 864000 static-gzip-all = True offload-threads = %k + +# Cache +cache2 = name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1 diff --git a/searx/shared/__init__.py b/searx/shared/__init__.py new file mode 100644 index 000000000..83d3a2742 --- /dev/null +++ b/searx/shared/__init__.py @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later + +import logging + +logger = logging.getLogger('searx.shared') + +try: + import uwsgi +except: + # no uwsgi + from .shared_simple import SimpleSharedDict as SharedDict, schedule + logger.info('Use shared_simple implementation') +else: + try: + uwsgi.cache_update('dummy', b'dummy') + if uwsgi.cache_get('dummy') != b'dummy': + raise Exception() + except: + # uwsgi.ini configuration problem: disable all scheduling + logger.error('uwsgi.ini configuration error, add this line to your uwsgi.ini\n' + 'cache2 = name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1') + from .shared_simple import SimpleSharedDict as SharedDict + + def schedule(delay, func, *args): + pass + else: + # uwsgi + from .shared_uwsgi import UwsgiCacheSharedDict as SharedDict, schedule + logger.info('Use shared_uwsgi implementation') + +storage = SharedDict() diff --git a/searx/shared/shared_abstract.py b/searx/shared/shared_abstract.py new file mode 100644 index 000000000..3fede417e --- /dev/null +++ b/searx/shared/shared_abstract.py @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later + +class SharedDict: + + def get_int(self, key): + pass + + def set_int(self, key, value): + pass + + def get_str(self, key): + pass + + def set_str(self, key, value): + pass diff --git a/searx/shared/shared_simple.py b/searx/shared/shared_simple.py new file mode 100644 index 000000000..5b970aad9 --- /dev/null +++ b/searx/shared/shared_simple.py @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later + +import threading + +from . import shared_abstract + + +class SimpleSharedDict(shared_abstract.SharedDict): + + __slots__ = 'd', + + def __init__(self): + self.d = {} + + def get_int(self, key): + return self.d.get(key, None) + + def set_int(self, key, value): + self.d[key] = value + + def get_str(self, key): + return self.d.get(key, None) + + def set_str(self, key, value): + self.d[key] = value + + +def schedule(delay, func, *args): + def call_later(): + t = threading.Timer(delay, wrapper) + t.daemon = True + t.start() + + def wrapper(): + call_later() + func(*args) + + call_later() diff --git a/searx/shared/shared_uwsgi.py b/searx/shared/shared_uwsgi.py new file mode 100644 index 000000000..136bf687e --- /dev/null +++ b/searx/shared/shared_uwsgi.py @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later + +import time +import uwsgi # pylint: disable=E0401 +from . import shared_abstract + + +_last_signal = 10 + + +class UwsgiCacheSharedDict(shared_abstract.SharedDict): + + def get_int(self, key): + value = uwsgi.cache_get(key) + if value is None: + return value + else: + return int.from_bytes(value, 'big') + + def set_int(self, key, value): + b = value.to_bytes(4, 'big') + uwsgi.cache_update(key, b) + + def get_str(self, key): + value = uwsgi.cache_get(key) + if value is None: + return value + else: + return value.decode('utf-8') + + def set_str(self, key, value): + b = value.encode('utf-8') + uwsgi.cache_update(key, b) + + +def schedule(delay, func, *args): + """ + Can be implemented using a spooler. + https://uwsgi-docs.readthedocs.io/en/latest/PythonDecorators.html + + To make the uwsgi configuration simple, use the alternative implementation. + """ + global _last_signal + + def sighandler(signum): + now = int(time.time()) + uwsgi.lock() + try: + updating = uwsgi.cache_get('updating') + if updating is not None: + updating = int.from_bytes(updating, 'big') + if now - updating < delay: + return + uwsgi.cache_update('updating', now.to_bytes(4, 'big')) + finally: + uwsgi.unlock() + func(*args) + + signal_num = _last_signal + _last_signal += 1 + uwsgi.register_signal(signal_num, 'worker', sighandler) + uwsgi.add_timer(signal_num, delay) diff --git a/utils/templates/etc/uwsgi/apps-archlinux/searx.ini b/utils/templates/etc/uwsgi/apps-archlinux/searx.ini index 9dd2e6f2f..71cece3c4 100644 --- a/utils/templates/etc/uwsgi/apps-archlinux/searx.ini +++ b/utils/templates/etc/uwsgi/apps-archlinux/searx.ini @@ -82,4 +82,7 @@ http = ${SEARX_INTERNAL_HTTP} # mkdir -p /run/uwsgi/app/searx # chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx # -# socket = /run/uwsgi/app/searx/socket
\ No newline at end of file +# socket = /run/uwsgi/app/searx/socket + +# Cache +cache2 = name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1 diff --git a/utils/templates/etc/uwsgi/apps-available/searx.ini b/utils/templates/etc/uwsgi/apps-available/searx.ini index 4d69da0cf..45214ef13 100644 --- a/utils/templates/etc/uwsgi/apps-available/searx.ini +++ b/utils/templates/etc/uwsgi/apps-available/searx.ini @@ -82,3 +82,6 @@ http = ${SEARX_INTERNAL_HTTP} # chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx # # socket = /run/uwsgi/app/searx/socket + +# Cache +cache2 = name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1 |