From 34e260f88f561cf26454fe72a10a3a403faf2ff3 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Fri, 11 Nov 2022 20:14:47 +0100 Subject: [fix] follow up of PR-1856 - the environment variable SEARXNG_REDIS_URL overrides the setting value redis.url - ./manage sets SEARXNG_REDIS_URL to unix:///usr/local/searxng-redis/run/redis.sock if: - the socket exists - SEARXNG_REDIS_URL is not already defined Update of PR #1856 Co-authored-by: Markus Heiser --- manage | 12 ++++++++++++ searx/settings_defaults.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/manage b/manage index c887826e2..d61237984 100755 --- a/manage +++ b/manage @@ -32,6 +32,12 @@ export NODE_MINIMUM_VERSION="16.13.0" BLACK_OPTIONS=("--target-version" "py37" "--line-length" "120" "--skip-string-normalization") BLACK_TARGETS=("--exclude" "searx/static,searx/languages.py" "--include" 'searxng.msg|\.pyi?$' "searx" "searxng_extra" "tests") +_dev_redis_sock="/usr/local/searxng-redis/run/redis.sock" +# set SEARXNG_REDIS_URL if it is not defined and "{_dev_redis_sock}" exists. +if [ -S "${_dev_redis_sock}" ] && [ -z "${SEARXNG_REDIS_URL}" ]; then + export SEARXNG_REDIS_URL="unix://${_dev_redis_sock}?db=0" +fi + pylint.FILES() { # List files tagged by comment: @@ -65,6 +71,8 @@ help() { cat < Date: Fri, 11 Nov 2022 21:58:32 +0100 Subject: move searx.shared.redisdb to searx.redisdb --- docs/admin/engines/settings.rst | 2 +- docs/admin/installation-uwsgi.rst | 4 +-- docs/admin/update-searxng.rst | 3 +- docs/src/searx.redisdb.rst | 8 +++++ docs/src/searx.shared.redisdb.rst | 8 ----- searx/plugins/limiter.py | 2 +- searx/redisdb.py | 70 ++++++++++++++++++++++++++++++++++++++ searx/search/checker/background.py | 2 +- searx/search/checker/scheduler.py | 2 +- searx/shared/__init__.py | 6 ---- searx/shared/redisdb.py | 70 -------------------------------------- searx/webapp.py | 2 +- utils/searxng.sh | 6 ++-- utils/searxng_check.py | 3 +- 14 files changed, 90 insertions(+), 98 deletions(-) create mode 100644 docs/src/searx.redisdb.rst delete mode 100644 docs/src/searx.shared.redisdb.rst create mode 100644 searx/redisdb.py delete mode 100644 searx/shared/__init__.py delete mode 100644 searx/shared/redisdb.py diff --git a/docs/admin/engines/settings.rst b/docs/admin/engines/settings.rst index 3ac992d3e..c747e3f43 100644 --- a/docs/admin/engines/settings.rst +++ b/docs/admin/engines/settings.rst @@ -294,7 +294,7 @@ Global Settings .. _Redis.from_url(url): https://redis-py.readthedocs.io/en/stable/connections.html#redis.client.Redis.from_url -A redis DB can be connected by an URL, in :py:obj:`searx.shared.redisdb` you +A redis DB can be connected by an URL, in :py:obj:`searx.redisdb` you will find a description to test your redis connection in SerXNG. When using sockets, don't forget to check the access rights on the socket:: diff --git a/docs/admin/installation-uwsgi.rst b/docs/admin/installation-uwsgi.rst index a6ea85608..e888d067e 100644 --- a/docs/admin/installation-uwsgi.rst +++ b/docs/admin/installation-uwsgi.rst @@ -239,8 +239,8 @@ For example on Fedora (RHEL): If you try to install a redis DB with socket communication and you want to connect to it from the SearXNG uWSGI, you will see a *Permission denied* in the log of your instance:: - ERROR:searx.shared.redis: [searxng (993)] can't connect redis DB ... - ERROR:searx.shared.redis: Error 13 connecting to unix socket: /usr/local/searxng-redis/run/redis.sock. Permission denied. + ERROR:searx.redisdb: [searxng (993)] can't connect redis DB ... + ERROR:searx.redisdb: Error 13 connecting to unix socket: /usr/local/searxng-redis/run/redis.sock. Permission denied. ERROR:searx.plugins.limiter: init limiter DB failed!!! Even if your *searxng* user of the uWSGI process is added to additional groups diff --git a/docs/admin/update-searxng.rst b/docs/admin/update-searxng.rst index 6c199638a..0d7b5e1b8 100644 --- a/docs/admin/update-searxng.rst +++ b/docs/admin/update-searxng.rst @@ -132,5 +132,4 @@ to see if there are some left overs. In this example there exists a *old* INFO: [OK] (old) account 'filtron' does not exists INFO: [OK] (old) account 'morty' does not exists ... - INFO searx.shared : Use shared_simple implementation - INFO searx.shared.redis : connected redis DB --> default + INFO searx.redisdb : connected redis DB --> default diff --git a/docs/src/searx.redisdb.rst b/docs/src/searx.redisdb.rst new file mode 100644 index 000000000..625378c91 --- /dev/null +++ b/docs/src/searx.redisdb.rst @@ -0,0 +1,8 @@ +.. _redis db: + +======== +Redis DB +======== + +.. automodule:: searx.redisdb + :members: diff --git a/docs/src/searx.shared.redisdb.rst b/docs/src/searx.shared.redisdb.rst deleted file mode 100644 index 265d87617..000000000 --- a/docs/src/searx.shared.redisdb.rst +++ /dev/null @@ -1,8 +0,0 @@ -.. _redis db: - -======== -Redis DB -======== - -.. automodule:: searx.shared.redisdb - :members: diff --git a/searx/plugins/limiter.py b/searx/plugins/limiter.py index c11fd506b..b66a0805c 100644 --- a/searx/plugins/limiter.py +++ b/searx/plugins/limiter.py @@ -16,7 +16,7 @@ Enable the plugin in ``settings.yml``: import re from flask import request -from searx.shared import redisdb +from searx import redisdb from searx.redislib import incr_sliding_window name = "Request limiter" diff --git a/searx/redisdb.py b/searx/redisdb.py new file mode 100644 index 000000000..0544d697f --- /dev/null +++ b/searx/redisdb.py @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# lint: pylint +"""Implementation of the redis client (redis-py_). + +.. _redis-py: https://github.com/redis/redis-py + +This implementation uses the :ref:`settings redis` setup from ``settings.yml``. +A redis DB connect can be tested by:: + + >>> from searx import redisdb + >>> redisdb.initialize() + True + >>> db = redisdb.client() + >>> db.set("foo", "bar") + True + >>> db.get("foo") + b'bar' + >>> + +""" + +import os +import pwd +import logging +import redis +from searx import get_setting + + +OLD_REDIS_URL_DEFAULT_URL = 'unix:///usr/local/searxng-redis/run/redis.sock?db=0' +"""This was the default Redis URL in settings.yml.""" + +_CLIENT = None +logger = logging.getLogger(__name__) + + +def client() -> redis.Redis: + return _CLIENT + + +def initialize(): + global _CLIENT # pylint: disable=global-statement + redis_url = get_setting('redis.url') + if not redis_url: + return False + try: + # create a client, but no connection is done + _CLIENT = redis.Redis.from_url(redis_url) + + # log the parameters as seen by the redis lib, without the password + kwargs = _CLIENT.get_connection_kwargs().copy() + kwargs.pop('password', None) + kwargs = ' '.join([f'{k}={v!r}' for k, v in kwargs.items()]) + logger.info("connecting to Redis %s", kwargs) + + # check the connection + _CLIENT.ping() + + # no error: the redis connection is working + logger.info("connected to Redis") + return True + except redis.exceptions.RedisError as e: + _CLIENT = None + _pw = pwd.getpwuid(os.getuid()) + logger.exception("[%s (%s)] can't connect redis DB ...", _pw.pw_name, _pw.pw_uid) + if redis_url == OLD_REDIS_URL_DEFAULT_URL and isinstance(e, redis.exceptions.ConnectionError): + logger.info( + "You can safely ignore the above Redis error if you don't use Redis. " + "You can remove this error by setting redis.url to false in your settings.yml." + ) + return False diff --git a/searx/search/checker/background.py b/searx/search/checker/background.py index e5bd642c0..aec2a1790 100644 --- a/searx/search/checker/background.py +++ b/searx/search/checker/background.py @@ -14,7 +14,7 @@ from typing_extensions import TypedDict, Literal import redis.exceptions from searx import logger, settings, searx_debug -from searx.shared.redisdb import client as get_redis_client +from searx.redisdb import client as get_redis_client from searx.exceptions import SearxSettingsException from searx.search.processors import PROCESSORS from searx.search.checker import Checker diff --git a/searx/search/checker/scheduler.py b/searx/search/checker/scheduler.py index 1ae635951..cc3bb7380 100644 --- a/searx/search/checker/scheduler.py +++ b/searx/search/checker/scheduler.py @@ -17,7 +17,7 @@ import time import importlib from typing import Callable -from searx.shared.redisdb import client as get_redis_client +from searx.redisdb import client as get_redis_client from searx.redislib import lua_script_storage diff --git a/searx/shared/__init__.py b/searx/shared/__init__.py deleted file mode 100644 index 2c7fc9f8b..000000000 --- a/searx/shared/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-or-later -# lint: pylint -"""Initialization of a *shared* storage. -""" - -from . import redisdb diff --git a/searx/shared/redisdb.py b/searx/shared/redisdb.py deleted file mode 100644 index d8e29d911..000000000 --- a/searx/shared/redisdb.py +++ /dev/null @@ -1,70 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-or-later -# lint: pylint -"""Implementation of the redis client (redis-py_). - -.. _redis-py: https://github.com/redis/redis-py - -This implementation uses the :ref:`settings redis` setup from ``settings.yml``. -A redis DB connect can be tested by:: - - >>> from searx.shared import redisdb - >>> redisdb.init() - True - >>> db = redisdb.client() - >>> db.set("foo", "bar") - True - >>> db.get("foo") - b'bar' - >>> - -""" - -import os -import pwd -import logging -import redis -from searx import get_setting - - -OLD_REDIS_URL_DEFAULT_URL = 'unix:///usr/local/searxng-redis/run/redis.sock?db=0' -"""This was the default Redis URL in settings.yml.""" - -_CLIENT = None -logger = logging.getLogger('searx.shared.redisdb') - - -def client() -> redis.Redis: - return _CLIENT - - -def initialize(): - global _CLIENT # pylint: disable=global-statement - redis_url = get_setting('redis.url') - if not redis_url: - return False - try: - # create a client, but no connection is done - _CLIENT = redis.Redis.from_url(redis_url) - - # log the parameters as seen by the redis lib, without the password - kwargs = _CLIENT.get_connection_kwargs().copy() - kwargs.pop('password', None) - kwargs = ' '.join([f'{k}={v!r}' for k, v in kwargs.items()]) - logger.info("connecting to Redis %s", kwargs) - - # check the connection - _CLIENT.ping() - - # no error: the redis connection is working - logger.info("connected to Redis") - return True - except redis.exceptions.RedisError as e: - _CLIENT = None - _pw = pwd.getpwuid(os.getuid()) - logger.exception("[%s (%s)] can't connect redis DB ...", _pw.pw_name, _pw.pw_uid) - if redis_url == OLD_REDIS_URL_DEFAULT_URL and isinstance(e, redis.exceptions.ConnectionError): - logger.info( - "You can safely ignore the above Redis error if you don't use Redis. " - "You can remove this error by setting redis.url to false in your settings.yml." - ) - return False diff --git a/searx/webapp.py b/searx/webapp.py index 4f334a9d0..d4206ca16 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -120,7 +120,7 @@ from searx.locales import ( # renaming names from searx imports ... from searx.autocomplete import search_autocomplete, backends as autocomplete_backends from searx.languages import language_codes as languages -from searx.shared.redisdb import initialize as redis_initialize +from searx.redisdb import initialize as redis_initialize from searx.search import SearchWithPlugins, initialize as search_initialize from searx.network import stream as http_stream, set_context_network_name from searx.search.checker import get_result as checker_get_result diff --git a/utils/searxng.sh b/utils/searxng.sh index e7aa8ed15..aaa69cc53 100755 --- a/utils/searxng.sh +++ b/utils/searxng.sh @@ -295,7 +295,7 @@ In your instance, redis DB connector is configured at: ${redis_url} " - if searxng.instance.exec python -c "from searx.shared import redisdb; redisdb.initialize() or exit(42)"; then + if searxng.instance.exec python -c "from searx import redisdb; redisdb.initialize() or exit(42)"; then info_msg "SearXNG instance is able to connect redis DB." return fi @@ -317,8 +317,8 @@ In your instance, redis DB connector is configured at: # fedora35 there is v2.0.20 installed --> no way to get additional # groups on fedora's tyrant mode. # - # ERROR:searx.shared.redis: [searxng (993)] can't connect redis DB ... - # ERROR:searx.shared.redis: Error 13 connecting to unix socket: /usr/local/searxng-redis/run/redis.sock. Permission denied. + # ERROR:searx.redisdb: [searxng (993)] can't connect redis DB ... + # ERROR:searx.redisdb: Error 13 connecting to unix socket: /usr/local/searxng-redis/run/redis.sock. Permission denied. # ERROR:searx.plugins.limiter: init limiter DB failed!!! # # $ ps -aef | grep '/usr/sbin/uwsgi --ini searxng.ini' diff --git a/utils/searxng_check.py b/utils/searxng_check.py index bd2d60288..dca4f1cda 100644 --- a/utils/searxng_check.py +++ b/utils/searxng_check.py @@ -26,8 +26,7 @@ if os.path.isfile(OLD_SETTING): )) warnings.warn(msg, DeprecationWarning) -from searx.shared import redisdb -from searx import get_setting +from searx import redisdb, get_setting if not redisdb.initialize(): warnings.warn("can't connect to redis DB at: %s" % get_setting('redis.url'), RuntimeWarning, stacklevel=2) -- cgit v1.2.3-54-g00ecf From 3050e2b6e875ccfb0162e83075802467c527c7fe Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Sat, 10 Dec 2022 10:06:54 +0100 Subject: [fix] documentation about update-searxng.rst --- docs/admin/update-searxng.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/admin/update-searxng.rst b/docs/admin/update-searxng.rst index 0d7b5e1b8..3ddb41b59 100644 --- a/docs/admin/update-searxng.rst +++ b/docs/admin/update-searxng.rst @@ -132,4 +132,5 @@ to see if there are some left overs. In this example there exists a *old* INFO: [OK] (old) account 'filtron' does not exists INFO: [OK] (old) account 'morty' does not exists ... - INFO searx.redisdb : connected redis DB --> default + INFO searx.redisdb : connecting to Redis db=0 path='/usr/local/searxng-redis/run/redis.sock' + INFO searx.redisdb : connected to Redis -- cgit v1.2.3-54-g00ecf