From fd814aac863673047c46a9d80682415dae180969 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 2 Oct 2023 16:36:07 +0200 Subject: [mod] isolation of botdetection from the limiter This patch was inspired by the discussion around PR-2882 [2]. The goals of this patch are: 1. Convert plugin searx.plugin.limiter to normal code [1] 2. isolation of botdetection from the limiter [2] 3. searx/{tools => botdetection}/config.py and drop searx.tools 4. in URL /config, 'limiter.enabled' is true only if the limiter is really enabled (Redis is available). This patch moves all the code that belongs to botdetection into namespace searx.botdetection and code that belongs to limiter is placed in namespace searx.limiter. Tthe limiter used to be a plugin at some point botdetection was added, it was not a plugin. The modularization of these two components was long overdue. With the clear modularization, the documentation could then also be organized according to the architecture. [1] https://github.com/searxng/searxng/pull/2882 [2] https://github.com/searxng/searxng/pull/2882#issuecomment-1741716891 To test: - check the app works without the limiter, check `/config` - check the app works with the limiter and with the token, check `/config` - make docs.live .. and read - http://0.0.0.0:8000/admin/searx.limiter.html - http://0.0.0.0:8000/src/searx.botdetection.html#botdetection Signed-off-by: Markus Heiser --- docs/admin/index.rst | 2 +- docs/admin/searx.botdetection.rst | 52 --------------------------- docs/admin/searx.limiter.rst | 17 +++++++++ docs/admin/settings/settings_server.rst | 4 +-- docs/conf.py | 5 ++- docs/src/searx.botdetection.rst | 62 +++++++++++++++++++++++++++++++++ 6 files changed, 85 insertions(+), 57 deletions(-) delete mode 100644 docs/admin/searx.botdetection.rst create mode 100644 docs/admin/searx.limiter.rst create mode 100644 docs/src/searx.botdetection.rst (limited to 'docs') diff --git a/docs/admin/index.rst b/docs/admin/index.rst index f335f1631..606b51c22 100644 --- a/docs/admin/index.rst +++ b/docs/admin/index.rst @@ -15,7 +15,7 @@ Administrator documentation installation-apache update-searxng answer-captcha - searx.botdetection + searx.limiter api architecture plugins diff --git a/docs/admin/searx.botdetection.rst b/docs/admin/searx.botdetection.rst deleted file mode 100644 index c89f1cd93..000000000 --- a/docs/admin/searx.botdetection.rst +++ /dev/null @@ -1,52 +0,0 @@ -.. _botdetection: - -============= -Bot Detection -============= - -.. contents:: - :depth: 2 - :local: - :backlinks: entry - -.. automodule:: searx.botdetection - :members: - -.. automodule:: searx.botdetection.limiter - :members: - -.. automodule:: searx.botdetection.ip_lists - :members: - - -.. _botdetection rate limit: - -Rate limit -========== - -.. automodule:: searx.botdetection.ip_limit - :members: - -.. automodule:: searx.botdetection.link_token - :members: - - -.. _botdetection probe headers: - -Probe HTTP headers -================== - -.. automodule:: searx.botdetection.http_accept - :members: - -.. automodule:: searx.botdetection.http_accept_encoding - :members: - -.. automodule:: searx.botdetection.http_accept_language - :members: - -.. automodule:: searx.botdetection.http_connection - :members: - -.. automodule:: searx.botdetection.http_user_agent - :members: diff --git a/docs/admin/searx.limiter.rst b/docs/admin/searx.limiter.rst new file mode 100644 index 000000000..c23635571 --- /dev/null +++ b/docs/admin/searx.limiter.rst @@ -0,0 +1,17 @@ +.. _limiter: + +======= +Limiter +======= + +.. sidebar:: info + + The limiter requires a :ref:`Redis ` database. + +.. contents:: + :depth: 2 + :local: + :backlinks: entry + +.. automodule:: searx.limiter + :members: diff --git a/docs/admin/settings/settings_server.rst b/docs/admin/settings/settings_server.rst index e2b4cb67d..ba0c9484d 100644 --- a/docs/admin/settings/settings_server.rst +++ b/docs/admin/settings/settings_server.rst @@ -36,11 +36,9 @@ ``secret_key`` : ``$SEARXNG_SECRET`` Used for cryptography purpose. -.. _limiter: - ``limiter`` : Rate limit the number of request on the instance, block some bots. The - :ref:`limiter src` requires a :ref:`settings redis` database. + :ref:`limiter` requires a :ref:`settings redis` database. .. _image_proxy: diff --git a/docs/conf.py b/docs/conf.py index aa4905ef4..2ed85a800 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: AGPL-3.0-or-later import sys, os +from pathlib import Path from pallets_sphinx_themes import ProjectLink from searx import get_setting @@ -13,7 +14,6 @@ project = 'SearXNG' copyright = 'SearXNG team' author = 'SearXNG team' release, version = VERSION_STRING, VERSION_STRING - SEARXNG_URL = get_setting('server.base_url') or 'https://example.org/searxng' ISSUE_URL = get_setting('brand.issue_url') DOCS_URL = get_setting('brand.docs_url') @@ -22,6 +22,9 @@ PRIVACYPOLICY_URL = get_setting('general.privacypolicy_url') CONTACT_URL = get_setting('general.contact_url') WIKI_URL = get_setting('brand.wiki_url') +SOURCEDIR = Path(__file__).parent.parent / "searx" +os.environ['SOURCEDIR'] = str(SOURCEDIR) + # hint: sphinx.ext.viewcode won't highlight when 'highlight_language' [1] is set # to string 'none' [2] # diff --git a/docs/src/searx.botdetection.rst b/docs/src/searx.botdetection.rst new file mode 100644 index 000000000..04cb81dfd --- /dev/null +++ b/docs/src/searx.botdetection.rst @@ -0,0 +1,62 @@ +.. _botdetection: + +============= +Bot Detection +============= + +.. contents:: + :depth: 2 + :local: + :backlinks: entry + +.. automodule:: searx.botdetection + :members: + +.. _botdetection ip_lists: + +IP lists +======== + +.. automodule:: searx.botdetection.ip_lists + :members: + + +.. _botdetection rate limit: + +Rate limit +========== + +.. automodule:: searx.botdetection.ip_limit + :members: + +.. automodule:: searx.botdetection.link_token + :members: + + +.. _botdetection probe headers: + +Probe HTTP headers +================== + +.. automodule:: searx.botdetection.http_accept + :members: + +.. automodule:: searx.botdetection.http_accept_encoding + :members: + +.. automodule:: searx.botdetection.http_accept_language + :members: + +.. automodule:: searx.botdetection.http_connection + :members: + +.. automodule:: searx.botdetection.http_user_agent + :members: + +.. _botdetection config: + +Config +====== + +.. automodule:: searx.botdetection.config + :members: -- cgit v1.2.3-54-g00ecf