[enh] add searx.shared

shared dictionary between the workers (UWSGI or werkzeug)
scheduler: run a task once every x seconds (UWSGI or werkzeug)

4 files changed, 146 insertions, 0 deletions

diff --git a/searx/shared/__init__.py b/searx/shared/__init__.py
new file mode 100644
index 000000000..83d3a2742
--- /dev/null
+++ b/searx/shared/__init__.py
@@ -0,0 +1,31 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+import logging
+
+logger = logging.getLogger('searx.shared')
+
+try:
+    import uwsgi
+except:
+    # no uwsgi
+    from .shared_simple import SimpleSharedDict as SharedDict, schedule
+    logger.info('Use shared_simple implementation')
+else:
+    try:
+        uwsgi.cache_update('dummy', b'dummy')
+        if uwsgi.cache_get('dummy') != b'dummy':
+            raise Exception()
+    except:
+        # uwsgi.ini configuration problem: disable all scheduling
+        logger.error('uwsgi.ini configuration error, add this line to your uwsgi.ini\n'
+                     'cache2 = name=searxcache,items=2000,blocks=2000,blocksize=4096,bitmap=1')
+        from .shared_simple import SimpleSharedDict as SharedDict
+
+        def schedule(delay, func, *args):
+            pass
+    else:
+        # uwsgi
+        from .shared_uwsgi import UwsgiCacheSharedDict as SharedDict, schedule
+        logger.info('Use shared_uwsgi implementation')
+
+storage = SharedDict()
diff --git a/searx/shared/shared_abstract.py b/searx/shared/shared_abstract.py
new file mode 100644
index 000000000..3fede417e
--- /dev/null
+++ b/searx/shared/shared_abstract.py
@@ -0,0 +1,15 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+class SharedDict:
+
+    def get_int(self, key):
+        pass
+
+    def set_int(self, key, value):
+        pass
+
+    def get_str(self, key):
+        pass
+
+    def set_str(self, key, value):
+        pass
diff --git a/searx/shared/shared_simple.py b/searx/shared/shared_simple.py
new file mode 100644
index 000000000..5b970aad9
--- /dev/null
+++ b/searx/shared/shared_simple.py
@@ -0,0 +1,38 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+import threading
+
+from . import shared_abstract
+
+
+class SimpleSharedDict(shared_abstract.SharedDict):
+
+    __slots__ = 'd',
+
+    def __init__(self):
+        self.d = {}
+
+    def get_int(self, key):
+        return self.d.get(key, None)
+
+    def set_int(self, key, value):
+        self.d[key] = value
+
+    def get_str(self, key):
+        return self.d.get(key, None)
+
+    def set_str(self, key, value):
+        self.d[key] = value
+
+
+def schedule(delay, func, *args):
+    def call_later():
+        t = threading.Timer(delay, wrapper)
+        t.daemon = True
+        t.start()
+
+    def wrapper():
+        call_later()
+        func(*args)
+
+    call_later()
diff --git a/searx/shared/shared_uwsgi.py b/searx/shared/shared_uwsgi.py
new file mode 100644
index 000000000..136bf687e
--- /dev/null
+++ b/searx/shared/shared_uwsgi.py
@@ -0,0 +1,62 @@
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+import time
+import uwsgi  # pylint: disable=E0401
+from . import shared_abstract
+
+
+_last_signal = 10
+
+
+class UwsgiCacheSharedDict(shared_abstract.SharedDict):
+
+    def get_int(self, key):
+        value = uwsgi.cache_get(key)
+        if value is None:
+            return value
+        else:
+            return int.from_bytes(value, 'big')
+
+    def set_int(self, key, value):
+        b = value.to_bytes(4, 'big')
+        uwsgi.cache_update(key, b)
+
+    def get_str(self, key):
+        value = uwsgi.cache_get(key)
+        if value is None:
+            return value
+        else:
+            return value.decode('utf-8')
+
+    def set_str(self, key, value):
+        b = value.encode('utf-8')
+        uwsgi.cache_update(key, b)
+
+
+def schedule(delay, func, *args):
+    """
+    Can be implemented using a spooler.
+    https://uwsgi-docs.readthedocs.io/en/latest/PythonDecorators.html
+
+    To make the uwsgi configuration simple, use the alternative implementation.
+    """
+    global _last_signal
+
+    def sighandler(signum):
+        now = int(time.time())
+        uwsgi.lock()
+        try:
+            updating = uwsgi.cache_get('updating')
+            if updating is not None:
+                updating = int.from_bytes(updating, 'big')
+                if now - updating < delay:
+                    return
+            uwsgi.cache_update('updating', now.to_bytes(4, 'big'))
+        finally:
+            uwsgi.unlock()
+        func(*args)
+
+    signal_num = _last_signal
+    _last_signal += 1
+    uwsgi.register_signal(signal_num, 'worker', sighandler)
+    uwsgi.add_timer(signal_num, delay)