diff options
Diffstat (limited to '_modules/searx/redislib.html')
| -rw-r--r-- | _modules/searx/redislib.html | 362 |
1 files changed, 362 insertions, 0 deletions
diff --git a/_modules/searx/redislib.html b/_modules/searx/redislib.html new file mode 100644 index 000000000..b130fd2ce --- /dev/null +++ b/_modules/searx/redislib.html @@ -0,0 +1,362 @@ +<!DOCTYPE html> + +<html lang="en" data-content_root="../../"> + <head> + <meta charset="utf-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <title>searx.redislib — SearXNG Documentation (2025.1.6+6dab7fe78)</title> + <link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=4f649999" /> + <link rel="stylesheet" type="text/css" href="../../_static/searxng.css?v=52e4ff28" /> + <script src="../../_static/documentation_options.js?v=ef740023"></script> + <script src="../../_static/doctools.js?v=9a2dae69"></script> + <script src="../../_static/sphinx_highlight.js?v=dc90522c"></script> + <script data-project="searxng" data-version="2025.1.6+6dab7fe78" src="../../_static/describe_version.js?v=fa7f30d0"></script> + <link rel="index" title="Index" href="../../genindex.html" /> + <link rel="search" title="Search" href="../../search.html" /> + </head><body> + <div class="related" role="navigation" aria-label="Related"> + <h3>Navigation</h3> + <ul> + <li class="right" style="margin-right: 10px"> + <a href="../../genindex.html" title="General Index" + accesskey="I">index</a></li> + <li class="right" > + <a href="../../py-modindex.html" title="Python Module Index" + >modules</a> |</li> + <li class="nav-item nav-item-0"><a href="../../index.html">SearXNG Documentation (2025.1.6+6dab7fe78)</a> »</li> + <li class="nav-item nav-item-1"><a href="../index.html" accesskey="U">Module code</a> »</li> + <li class="nav-item nav-item-this"><a href="">searx.redislib</a></li> + </ul> + </div> + + <div class="document"> + <div class="documentwrapper"> + <div class="bodywrapper"> + <div class="body" role="main"> + + <h1>Source code for searx.redislib</h1><div class="highlight"><pre> +<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span> +<span class="sd">"""A collection of convenient functions and redis/lua scripts.</span> + +<span class="sd">This code was partial inspired by the `Bullet-Proofing Lua Scripts in RedisPy`_</span> +<span class="sd">article.</span> + +<span class="sd">.. _Bullet-Proofing Lua Scripts in RedisPy:</span> +<span class="sd"> https://redis.com/blog/bullet-proofing-lua-scripts-in-redispy/</span> + +<span class="sd">"""</span> + +<span class="kn">import</span> <span class="nn">hmac</span> + +<span class="kn">from</span> <span class="nn">searx</span> <span class="kn">import</span> <span class="n">get_setting</span> + +<span class="n">LUA_SCRIPT_STORAGE</span> <span class="o">=</span> <span class="p">{}</span> +<span class="sd">"""A global dictionary to cache client's ``Script`` objects, used by</span> +<span class="sd">:py:obj:`lua_script_storage`"""</span> + + +<div class="viewcode-block" id="lua_script_storage"> +<a class="viewcode-back" href="../../src/searx.redislib.html#searx.redislib.lua_script_storage">[docs]</a> +<span class="k">def</span> <span class="nf">lua_script_storage</span><span class="p">(</span><span class="n">client</span><span class="p">,</span> <span class="n">script</span><span class="p">):</span> +<span class="w"> </span><span class="sd">"""Returns a redis :py:obj:`Script</span> +<span class="sd"> <redis.commands.core.CoreCommands.register_script>` instance.</span> + +<span class="sd"> Due to performance reason the ``Script`` object is instantiated only once</span> +<span class="sd"> for a client (``client.register_script(..)``) and is cached in</span> +<span class="sd"> :py:obj:`LUA_SCRIPT_STORAGE`.</span> + +<span class="sd"> """</span> + + <span class="c1"># redis connection can be closed, lets use the id() of the redis connector</span> + <span class="c1"># as key in the script-storage:</span> + <span class="n">client_id</span> <span class="o">=</span> <span class="nb">id</span><span class="p">(</span><span class="n">client</span><span class="p">)</span> + + <span class="k">if</span> <span class="n">LUA_SCRIPT_STORAGE</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">client_id</span><span class="p">)</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> + <span class="n">LUA_SCRIPT_STORAGE</span><span class="p">[</span><span class="n">client_id</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span> + + <span class="k">if</span> <span class="n">LUA_SCRIPT_STORAGE</span><span class="p">[</span><span class="n">client_id</span><span class="p">]</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">script</span><span class="p">)</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> + <span class="n">LUA_SCRIPT_STORAGE</span><span class="p">[</span><span class="n">client_id</span><span class="p">][</span><span class="n">script</span><span class="p">]</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">register_script</span><span class="p">(</span><span class="n">script</span><span class="p">)</span> + + <span class="k">return</span> <span class="n">LUA_SCRIPT_STORAGE</span><span class="p">[</span><span class="n">client_id</span><span class="p">][</span><span class="n">script</span><span class="p">]</span></div> + + + +<span class="n">PURGE_BY_PREFIX</span> <span class="o">=</span> <span class="s2">"""</span> +<span class="s2">local prefix = tostring(ARGV[1])</span> +<span class="s2">for i, name in ipairs(redis.call('KEYS', prefix .. '*')) do</span> +<span class="s2"> redis.call('EXPIRE', name, 0)</span> +<span class="s2">end</span> +<span class="s2">"""</span> + + +<div class="viewcode-block" id="purge_by_prefix"> +<a class="viewcode-back" href="../../src/searx.redislib.html#searx.redislib.purge_by_prefix">[docs]</a> +<span class="k">def</span> <span class="nf">purge_by_prefix</span><span class="p">(</span><span class="n">client</span><span class="p">,</span> <span class="n">prefix</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"SearXNG_"</span><span class="p">):</span> +<span class="w"> </span><span class="sd">"""Purge all keys with ``prefix`` from database.</span> + +<span class="sd"> Queries all keys in the database by the given prefix and set expire time to</span> +<span class="sd"> zero. The default prefix will drop all keys which has been set by SearXNG</span> +<span class="sd"> (drops SearXNG schema entirely from database).</span> + +<span class="sd"> The implementation is the lua script from string :py:obj:`PURGE_BY_PREFIX`.</span> +<span class="sd"> The lua script uses EXPIRE_ instead of DEL_: if there are a lot keys to</span> +<span class="sd"> delete and/or their values are big, `DEL` could take more time and blocks</span> +<span class="sd"> the command loop while `EXPIRE` turns back immediate.</span> + +<span class="sd"> :param prefix: prefix of the key to delete (default: ``SearXNG_``)</span> +<span class="sd"> :type name: str</span> + +<span class="sd"> .. _EXPIRE: https://redis.io/commands/expire/</span> +<span class="sd"> .. _DEL: https://redis.io/commands/del/</span> + +<span class="sd"> """</span> + <span class="n">script</span> <span class="o">=</span> <span class="n">lua_script_storage</span><span class="p">(</span><span class="n">client</span><span class="p">,</span> <span class="n">PURGE_BY_PREFIX</span><span class="p">)</span> + <span class="n">script</span><span class="p">(</span><span class="n">args</span><span class="o">=</span><span class="p">[</span><span class="n">prefix</span><span class="p">])</span></div> + + + +<div class="viewcode-block" id="secret_hash"> +<a class="viewcode-back" href="../../src/searx.redislib.html#searx.redislib.secret_hash">[docs]</a> +<span class="k">def</span> <span class="nf">secret_hash</span><span class="p">(</span><span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span> +<span class="w"> </span><span class="sd">"""Creates a hash of the ``name``.</span> + +<span class="sd"> Combines argument ``name`` with the ``secret_key`` from :ref:`settings</span> +<span class="sd"> server`. This function can be used to get a more anonymized name of a Redis</span> +<span class="sd"> KEY.</span> + +<span class="sd"> :param name: the name to create a secret hash for</span> +<span class="sd"> :type name: str</span> +<span class="sd"> """</span> + <span class="n">m</span> <span class="o">=</span> <span class="n">hmac</span><span class="o">.</span><span class="n">new</span><span class="p">(</span><span class="nb">bytes</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">'utf-8'</span><span class="p">),</span> <span class="n">digestmod</span><span class="o">=</span><span class="s1">'sha256'</span><span class="p">)</span> + <span class="n">m</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="nb">bytes</span><span class="p">(</span><span class="n">get_setting</span><span class="p">(</span><span class="s1">'server.secret_key'</span><span class="p">),</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">'utf-8'</span><span class="p">))</span> + <span class="k">return</span> <span class="n">m</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">()</span></div> + + + +<span class="n">INCR_COUNTER</span> <span class="o">=</span> <span class="s2">"""</span> +<span class="s2">local limit = tonumber(ARGV[1])</span> +<span class="s2">local expire = tonumber(ARGV[2])</span> +<span class="s2">local c_name = KEYS[1]</span> + +<span class="s2">local c = redis.call('GET', c_name)</span> + +<span class="s2">if not c then</span> +<span class="s2"> c = redis.call('INCR', c_name)</span> +<span class="s2"> if expire > 0 then</span> +<span class="s2"> redis.call('EXPIRE', c_name, expire)</span> +<span class="s2"> end</span> +<span class="s2">else</span> +<span class="s2"> c = tonumber(c)</span> +<span class="s2"> if limit == 0 or c < limit then</span> +<span class="s2"> c = redis.call('INCR', c_name)</span> +<span class="s2"> end</span> +<span class="s2">end</span> +<span class="s2">return c</span> +<span class="s2">"""</span> + + +<div class="viewcode-block" id="incr_counter"> +<a class="viewcode-back" href="../../src/searx.redislib.html#searx.redislib.incr_counter">[docs]</a> +<span class="k">def</span> <span class="nf">incr_counter</span><span class="p">(</span><span class="n">client</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">limit</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span> <span class="n">expire</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">):</span> +<span class="w"> </span><span class="sd">"""Increment a counter and return the new value.</span> + +<span class="sd"> If counter with redis key ``SearXNG_counter_<name>`` does not exists it is</span> +<span class="sd"> created with initial value 1 returned. The replacement ``<name>`` is a</span> +<span class="sd"> *secret hash* of the value from argument ``name`` (see</span> +<span class="sd"> :py:func:`secret_hash`).</span> + +<span class="sd"> The implementation of the redis counter is the lua script from string</span> +<span class="sd"> :py:obj:`INCR_COUNTER`.</span> + +<span class="sd"> :param name: name of the counter</span> +<span class="sd"> :type name: str</span> + +<span class="sd"> :param expire: live-time of the counter in seconds (default ``None`` means</span> +<span class="sd"> infinite).</span> +<span class="sd"> :type expire: int / see EXPIRE_</span> + +<span class="sd"> :param limit: limit where the counter stops to increment (default ``None``)</span> +<span class="sd"> :type limit: int / limit is 2^64 see INCR_</span> + +<span class="sd"> :return: value of the incremented counter</span> +<span class="sd"> :type return: int</span> + +<span class="sd"> .. _EXPIRE: https://redis.io/commands/expire/</span> +<span class="sd"> .. _INCR: https://redis.io/commands/incr/</span> + +<span class="sd"> A simple demo of a counter with expire time and limit::</span> + +<span class="sd"> >>> for i in range(6):</span> +<span class="sd"> ... i, incr_counter(client, "foo", 3, 5) # max 3, duration 5 sec</span> +<span class="sd"> ... time.sleep(1) # from the third call on max has been reached</span> +<span class="sd"> ...</span> +<span class="sd"> (0, 1)</span> +<span class="sd"> (1, 2)</span> +<span class="sd"> (2, 3)</span> +<span class="sd"> (3, 3)</span> +<span class="sd"> (4, 3)</span> +<span class="sd"> (5, 1)</span> + +<span class="sd"> """</span> + <span class="n">script</span> <span class="o">=</span> <span class="n">lua_script_storage</span><span class="p">(</span><span class="n">client</span><span class="p">,</span> <span class="n">INCR_COUNTER</span><span class="p">)</span> + <span class="n">name</span> <span class="o">=</span> <span class="s2">"SearXNG_counter_"</span> <span class="o">+</span> <span class="n">secret_hash</span><span class="p">(</span><span class="n">name</span><span class="p">)</span> + <span class="n">c</span> <span class="o">=</span> <span class="n">script</span><span class="p">(</span><span class="n">args</span><span class="o">=</span><span class="p">[</span><span class="n">limit</span><span class="p">,</span> <span class="n">expire</span><span class="p">],</span> <span class="n">keys</span><span class="o">=</span><span class="p">[</span><span class="n">name</span><span class="p">])</span> + <span class="k">return</span> <span class="n">c</span></div> + + + +<div class="viewcode-block" id="drop_counter"> +<a class="viewcode-back" href="../../src/searx.redislib.html#searx.redislib.drop_counter">[docs]</a> +<span class="k">def</span> <span class="nf">drop_counter</span><span class="p">(</span><span class="n">client</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span> +<span class="w"> </span><span class="sd">"""Drop counter with redis key ``SearXNG_counter_<name>``</span> + +<span class="sd"> The replacement ``<name>`` is a *secret hash* of the value from argument</span> +<span class="sd"> ``name`` (see :py:func:`incr_counter` and :py:func:`incr_sliding_window`).</span> +<span class="sd"> """</span> + <span class="n">name</span> <span class="o">=</span> <span class="s2">"SearXNG_counter_"</span> <span class="o">+</span> <span class="n">secret_hash</span><span class="p">(</span><span class="n">name</span><span class="p">)</span> + <span class="n">client</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="n">name</span><span class="p">)</span></div> + + + +<span class="n">INCR_SLIDING_WINDOW</span> <span class="o">=</span> <span class="s2">"""</span> +<span class="s2">local expire = tonumber(ARGV[1])</span> +<span class="s2">local name = KEYS[1]</span> +<span class="s2">local current_time = redis.call('TIME')</span> + +<span class="s2">redis.call('ZREMRANGEBYSCORE', name, 0, current_time[1] - expire)</span> +<span class="s2">redis.call('ZADD', name, current_time[1], current_time[1] .. current_time[2])</span> +<span class="s2">local result = redis.call('ZCOUNT', name, 0, current_time[1] + 1)</span> +<span class="s2">redis.call('EXPIRE', name, expire)</span> +<span class="s2">return result</span> +<span class="s2">"""</span> + + +<div class="viewcode-block" id="incr_sliding_window"> +<a class="viewcode-back" href="../../src/searx.redislib.html#searx.redislib.incr_sliding_window">[docs]</a> +<span class="k">def</span> <span class="nf">incr_sliding_window</span><span class="p">(</span><span class="n">client</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">duration</span><span class="p">:</span> <span class="nb">int</span><span class="p">):</span> +<span class="w"> </span><span class="sd">"""Increment a sliding-window counter and return the new value.</span> + +<span class="sd"> If counter with redis key ``SearXNG_counter_<name>`` does not exists it is</span> +<span class="sd"> created with initial value 1 returned. The replacement ``<name>`` is a</span> +<span class="sd"> *secret hash* of the value from argument ``name`` (see</span> +<span class="sd"> :py:func:`secret_hash`).</span> + +<span class="sd"> :param name: name of the counter</span> +<span class="sd"> :type name: str</span> + +<span class="sd"> :param duration: live-time of the sliding window in seconds</span> +<span class="sd"> :typeduration: int</span> + +<span class="sd"> :return: value of the incremented counter</span> +<span class="sd"> :type return: int</span> + +<span class="sd"> The implementation of the redis counter is the lua script from string</span> +<span class="sd"> :py:obj:`INCR_SLIDING_WINDOW`. The lua script uses `sorted sets in Redis`_</span> +<span class="sd"> to implement a sliding window for the redis key ``SearXNG_counter_<name>``</span> +<span class="sd"> (ZADD_). The current TIME_ is used to score the items in the sorted set and</span> +<span class="sd"> the time window is moved by removing items with a score lower current time</span> +<span class="sd"> minus *duration* time (ZREMRANGEBYSCORE_).</span> + +<span class="sd"> The EXPIRE_ time (the duration of the sliding window) is refreshed on each</span> +<span class="sd"> call (increment) and if there is no call in this duration, the sorted</span> +<span class="sd"> set expires from the redis DB.</span> + +<span class="sd"> The return value is the amount of items in the sorted set (ZCOUNT_), what</span> +<span class="sd"> means the number of calls in the sliding window.</span> + +<span class="sd"> .. _Sorted sets in Redis:</span> +<span class="sd"> https://redis.com/ebook/part-1-getting-started/chapter-1-getting-to-know-redis/1-2-what-redis-data-structures-look-like/1-2-5-sorted-sets-in-redis/</span> +<span class="sd"> .. _TIME: https://redis.io/commands/time/</span> +<span class="sd"> .. _ZADD: https://redis.io/commands/zadd/</span> +<span class="sd"> .. _EXPIRE: https://redis.io/commands/expire/</span> +<span class="sd"> .. _ZREMRANGEBYSCORE: https://redis.io/commands/zremrangebyscore/</span> +<span class="sd"> .. _ZCOUNT: https://redis.io/commands/zcount/</span> + +<span class="sd"> A simple demo of the sliding window::</span> + +<span class="sd"> >>> for i in range(5):</span> +<span class="sd"> ... incr_sliding_window(client, "foo", 3) # duration 3 sec</span> +<span class="sd"> ... time.sleep(1) # from the third call (second) on the window is moved</span> +<span class="sd"> ...</span> +<span class="sd"> 1</span> +<span class="sd"> 2</span> +<span class="sd"> 3</span> +<span class="sd"> 3</span> +<span class="sd"> 3</span> +<span class="sd"> >>> time.sleep(3) # wait until expire</span> +<span class="sd"> >>> incr_sliding_window(client, "foo", 3)</span> +<span class="sd"> 1</span> + +<span class="sd"> """</span> + <span class="n">script</span> <span class="o">=</span> <span class="n">lua_script_storage</span><span class="p">(</span><span class="n">client</span><span class="p">,</span> <span class="n">INCR_SLIDING_WINDOW</span><span class="p">)</span> + <span class="n">name</span> <span class="o">=</span> <span class="s2">"SearXNG_counter_"</span> <span class="o">+</span> <span class="n">secret_hash</span><span class="p">(</span><span class="n">name</span><span class="p">)</span> + <span class="n">c</span> <span class="o">=</span> <span class="n">script</span><span class="p">(</span><span class="n">args</span><span class="o">=</span><span class="p">[</span><span class="n">duration</span><span class="p">],</span> <span class="n">keys</span><span class="o">=</span><span class="p">[</span><span class="n">name</span><span class="p">])</span> + <span class="k">return</span> <span class="n">c</span></div> + +</pre></div> + + <div class="clearer"></div> + </div> + </div> + </div> + <span id="sidebar-top"></span> + <div class="sphinxsidebar" role="navigation" aria-label="Main"> + <div class="sphinxsidebarwrapper"> + + + <p class="logo"><a href="../../index.html"> + <img class="logo" src="../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/> + </a></p> + + +<h3><a href="../../index.html">Table of Contents</a></h3> +<ul> +<li class="toctree-l1"><a class="reference internal" href="../../user/index.html">User information</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../own-instance.html">Why use a private instance?</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../admin/index.html">Administrator documentation</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../dev/index.html">Developer documentation</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../utils/index.html">DevOps tooling box</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../src/index.html">Source-Code</a></li> +</ul> + + <h3>Project Links</h3> + <ul> + <li><a href="https://github.com/searxng/searxng/tree/master">Source</a> + + <li><a href="https://github.com/searxng/searxng/wiki">Wiki</a> + + <li><a href="https://searx.space">Public instances</a> + + <li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a> + </ul><h3>Navigation</h3> +<ul> + <li><a href="../../index.html">Overview</a> + <ul> + <li><a href="../index.html">Module code</a> + + + </ul> + </li> + </ul> + </li> +</ul> +<search id="searchbox" style="display: none" role="search"> + <h3 id="searchlabel">Quick search</h3> + <div class="searchformwrapper"> + <form class="search" action="../../search.html" method="get"> + <input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/> + <input type="submit" value="Go" /> + </form> + </div> +</search> +<script>document.getElementById('searchbox').style.display = "block"</script> + </div> + </div> + <div class="clearer"></div> + </div> + <div class="footer" role="contentinfo"> + © Copyright SearXNG team. + </div> + </body> +</html>
\ No newline at end of file |