diff options
Diffstat (limited to '_modules/searx/favicons')
| -rw-r--r-- | _modules/searx/favicons/cache.html | 643 | ||||
| -rw-r--r-- | _modules/searx/favicons/config.html | 175 | ||||
| -rw-r--r-- | _modules/searx/favicons/proxy.html | 362 | ||||
| -rw-r--r-- | _modules/searx/favicons/resolvers.html | 216 |
4 files changed, 1396 insertions, 0 deletions
diff --git a/_modules/searx/favicons/cache.html b/_modules/searx/favicons/cache.html new file mode 100644 index 000000000..f56d4b69c --- /dev/null +++ b/_modules/searx/favicons/cache.html @@ -0,0 +1,643 @@ +<!DOCTYPE html> + +<html lang="en" data-content_root="../../../"> + <head> + <meta charset="utf-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <title>searx.favicons.cache — SearXNG Documentation (2025.1.6+6dab7fe78)</title> + <link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=4f649999" /> + <link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" /> + <script src="../../../_static/documentation_options.js?v=ef740023"></script> + <script src="../../../_static/doctools.js?v=9a2dae69"></script> + <script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script> + <script data-project="searxng" data-version="2025.1.6+6dab7fe78" src="../../../_static/describe_version.js?v=fa7f30d0"></script> + <link rel="index" title="Index" href="../../../genindex.html" /> + <link rel="search" title="Search" href="../../../search.html" /> + </head><body> + <div class="related" role="navigation" aria-label="Related"> + <h3>Navigation</h3> + <ul> + <li class="right" style="margin-right: 10px"> + <a href="../../../genindex.html" title="General Index" + accesskey="I">index</a></li> + <li class="right" > + <a href="../../../py-modindex.html" title="Python Module Index" + >modules</a> |</li> + <li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2025.1.6+6dab7fe78)</a> »</li> + <li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li> + <li class="nav-item nav-item-this"><a href="">searx.favicons.cache</a></li> + </ul> + </div> + + <div class="document"> + <div class="documentwrapper"> + <div class="bodywrapper"> + <div class="body" role="main"> + + <h1>Source code for searx.favicons.cache</h1><div class="highlight"><pre> +<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span> +<span class="sd">"""Implementations for caching favicons.</span> + +<span class="sd">:py:obj:`FaviconCacheConfig`:</span> +<span class="sd"> Configuration of the favicon cache</span> + +<span class="sd">:py:obj:`FaviconCache`:</span> +<span class="sd"> Abstract base class for the implementation of a favicon cache.</span> + +<span class="sd">:py:obj:`FaviconCacheSQLite`:</span> +<span class="sd"> Favicon cache that manages the favicon BLOBs in a SQLite DB.</span> + +<span class="sd">:py:obj:`FaviconCacheNull`:</span> +<span class="sd"> Fallback solution if the configured cache cannot be used for system reasons.</span> + +<span class="sd">----</span> + +<span class="sd">"""</span> + +<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">annotations</span> +<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Literal</span> + +<span class="kn">import</span> <span class="nn">os</span> +<span class="kn">import</span> <span class="nn">abc</span> +<span class="kn">import</span> <span class="nn">dataclasses</span> +<span class="kn">import</span> <span class="nn">hashlib</span> +<span class="kn">import</span> <span class="nn">logging</span> +<span class="kn">import</span> <span class="nn">sqlite3</span> +<span class="kn">import</span> <span class="nn">tempfile</span> +<span class="kn">import</span> <span class="nn">time</span> +<span class="kn">import</span> <span class="nn">typer</span> + +<span class="kn">import</span> <span class="nn">msgspec</span> + +<span class="kn">from</span> <span class="nn">searx</span> <span class="kn">import</span> <span class="n">sqlitedb</span> +<span class="kn">from</span> <span class="nn">searx</span> <span class="kn">import</span> <span class="n">logger</span> +<span class="kn">from</span> <span class="nn">searx.utils</span> <span class="kn">import</span> <span class="n">humanize_bytes</span><span class="p">,</span> <span class="n">humanize_number</span> + +<span class="n">CACHE</span><span class="p">:</span> <span class="s2">"FaviconCache"</span> +<span class="n">FALLBACK_ICON</span> <span class="o">=</span> <span class="sa">b</span><span class="s2">"FALLBACK_ICON"</span> + +<span class="n">logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="s1">'favicons.cache'</span><span class="p">)</span> +<span class="n">app</span> <span class="o">=</span> <span class="n">typer</span><span class="o">.</span><span class="n">Typer</span><span class="p">()</span> + + +<div class="viewcode-block" id="state"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.state">[docs]</a> +<span class="nd">@app</span><span class="o">.</span><span class="n">command</span><span class="p">()</span> +<span class="k">def</span> <span class="nf">state</span><span class="p">():</span> +<span class="w"> </span><span class="sd">"""show state of the cache"""</span> + <span class="nb">print</span><span class="p">(</span><span class="n">CACHE</span><span class="o">.</span><span class="n">state</span><span class="p">()</span><span class="o">.</span><span class="n">report</span><span class="p">())</span></div> + + + +<div class="viewcode-block" id="maintenance"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.maintenance">[docs]</a> +<span class="nd">@app</span><span class="o">.</span><span class="n">command</span><span class="p">()</span> +<span class="k">def</span> <span class="nf">maintenance</span><span class="p">(</span><span class="n">force</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span> <span class="n">debug</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">):</span> +<span class="w"> </span><span class="sd">"""perform maintenance of the cache"""</span> + <span class="n">root_log</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">()</span> + <span class="k">if</span> <span class="n">debug</span><span class="p">:</span> + <span class="n">root_log</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">DEBUG</span><span class="p">)</span> + <span class="k">else</span><span class="p">:</span> + <span class="n">root_log</span><span class="o">.</span><span class="n">handlers</span> <span class="o">=</span> <span class="p">[]</span> + <span class="n">handler</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">StreamHandler</span><span class="p">()</span> + <span class="n">handler</span><span class="o">.</span><span class="n">setFormatter</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">Formatter</span><span class="p">(</span><span class="s2">"</span><span class="si">%(message)s</span><span class="s2">"</span><span class="p">))</span> + <span class="n">logger</span><span class="o">.</span><span class="n">addHandler</span><span class="p">(</span><span class="n">handler</span><span class="p">)</span> + <span class="n">logger</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">DEBUG</span><span class="p">)</span> + + <span class="n">state_t0</span> <span class="o">=</span> <span class="n">CACHE</span><span class="o">.</span><span class="n">state</span><span class="p">()</span> + <span class="n">CACHE</span><span class="o">.</span><span class="n">maintenance</span><span class="p">(</span><span class="n">force</span><span class="o">=</span><span class="n">force</span><span class="p">)</span> + <span class="n">state_t1</span> <span class="o">=</span> <span class="n">CACHE</span><span class="o">.</span><span class="n">state</span><span class="p">()</span> + <span class="n">state_delta</span> <span class="o">=</span> <span class="n">state_t0</span> <span class="o">-</span> <span class="n">state_t1</span> + <span class="nb">print</span><span class="p">(</span><span class="s2">"The cache has been reduced by:"</span><span class="p">)</span> + <span class="nb">print</span><span class="p">(</span><span class="n">state_delta</span><span class="o">.</span><span class="n">report</span><span class="p">(</span><span class="s2">"</span><span class="se">\n</span><span class="s2">- </span><span class="si">{descr}</span><span class="s2">: </span><span class="si">{val}</span><span class="s2">"</span><span class="p">)</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">))</span></div> + + + +<div class="viewcode-block" id="init"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.init">[docs]</a> +<span class="k">def</span> <span class="nf">init</span><span class="p">(</span><span class="n">cfg</span><span class="p">:</span> <span class="s2">"FaviconCacheConfig"</span><span class="p">):</span> +<span class="w"> </span><span class="sd">"""Initialization of a global ``CACHE``"""</span> + + <span class="k">global</span> <span class="n">CACHE</span> <span class="c1"># pylint: disable=global-statement</span> + <span class="k">if</span> <span class="n">cfg</span><span class="o">.</span><span class="n">db_type</span> <span class="o">==</span> <span class="s2">"sqlite"</span><span class="p">:</span> + <span class="k">if</span> <span class="n">sqlite3</span><span class="o">.</span><span class="n">sqlite_version_info</span> <span class="o"><=</span> <span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">35</span><span class="p">):</span> + <span class="n">logger</span><span class="o">.</span><span class="n">critical</span><span class="p">(</span> + <span class="s2">"Disable favicon caching completely: SQLite library (</span><span class="si">%s</span><span class="s2">) is too old! (require >= 3.35)"</span><span class="p">,</span> + <span class="n">sqlite3</span><span class="o">.</span><span class="n">sqlite_version</span><span class="p">,</span> + <span class="p">)</span> + <span class="n">CACHE</span> <span class="o">=</span> <span class="n">FaviconCacheNull</span><span class="p">(</span><span class="n">cfg</span><span class="p">)</span> + <span class="k">else</span><span class="p">:</span> + <span class="n">CACHE</span> <span class="o">=</span> <span class="n">FaviconCacheSQLite</span><span class="p">(</span><span class="n">cfg</span><span class="p">)</span> + <span class="k">elif</span> <span class="n">cfg</span><span class="o">.</span><span class="n">db_type</span> <span class="o">==</span> <span class="s2">"mem"</span><span class="p">:</span> + <span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Favicons are cached in memory, don't use this in production!"</span><span class="p">)</span> + <span class="n">CACHE</span> <span class="o">=</span> <span class="n">FaviconCacheMEM</span><span class="p">(</span><span class="n">cfg</span><span class="p">)</span> + <span class="k">else</span><span class="p">:</span> + <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"favicons db_type '</span><span class="si">{</span><span class="n">cfg</span><span class="o">.</span><span class="n">db_type</span><span class="si">}</span><span class="s2">' is unknown"</span><span class="p">)</span></div> + + + +<div class="viewcode-block" id="FaviconCacheConfig"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheConfig">[docs]</a> +<span class="k">class</span> <span class="nc">FaviconCacheConfig</span><span class="p">(</span><span class="n">msgspec</span><span class="o">.</span><span class="n">Struct</span><span class="p">):</span> <span class="c1"># pylint: disable=too-few-public-methods</span> +<span class="w"> </span><span class="sd">"""Configuration of the favicon cache."""</span> + + <span class="n">db_type</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">"sqlite"</span><span class="p">,</span> <span class="s2">"mem"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"sqlite"</span> +<span class="w"> </span><span class="sd">"""Type of the database:</span> + +<span class="sd"> ``sqlite``:</span> +<span class="sd"> :py:obj:`.cache.FaviconCacheSQLite`</span> + +<span class="sd"> ``mem``:</span> +<span class="sd"> :py:obj:`.cache.FaviconCacheMEM` (not recommended)</span> +<span class="sd"> """</span> + + <span class="n">db_url</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">gettempdir</span><span class="p">()</span> <span class="o">+</span> <span class="n">os</span><span class="o">.</span><span class="n">sep</span> <span class="o">+</span> <span class="s2">"faviconcache.db"</span> +<span class="w"> </span><span class="sd">"""URL of the SQLite DB, the path to the database file."""</span> + + <span class="n">HOLD_TIME</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">30</span> <span class="c1"># 30 days</span> +<span class="w"> </span><span class="sd">"""Hold time (default in sec.), after which a BLOB is removed from the cache."""</span> + + <span class="n">LIMIT_TOTAL_BYTES</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1024</span> <span class="o">*</span> <span class="mi">1024</span> <span class="o">*</span> <span class="mi">50</span> <span class="c1"># 50 MB</span> +<span class="w"> </span><span class="sd">"""Maximum of bytes (default) stored in the cache of all blobs. Note: The</span> +<span class="sd"> limit is only reached at each maintenance interval after which the oldest</span> +<span class="sd"> BLOBs are deleted; the limit is exceeded during the maintenance period. If</span> +<span class="sd"> the maintenance period is *too long* or maintenance is switched off</span> +<span class="sd"> completely, the cache grows uncontrollably."""</span> + + <span class="n">BLOB_MAX_BYTES</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1024</span> <span class="o">*</span> <span class="mi">20</span> <span class="c1"># 20 KB</span> +<span class="w"> </span><span class="sd">"""The maximum BLOB size in bytes that a favicon may have so that it can be</span> +<span class="sd"> saved in the cache. If the favicon is larger, it is not saved in the cache</span> +<span class="sd"> and must be requested by the client via the proxy."""</span> + + <span class="n">MAINTENANCE_PERIOD</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">60</span> +<span class="w"> </span><span class="sd">"""Maintenance period in seconds / when :py:obj:`MAINTENANCE_MODE` is set to</span> +<span class="sd"> ``auto``."""</span> + + <span class="n">MAINTENANCE_MODE</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">"auto"</span><span class="p">,</span> <span class="s2">"off"</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"auto"</span> +<span class="w"> </span><span class="sd">"""Type of maintenance mode</span> + +<span class="sd"> ``auto``:</span> +<span class="sd"> Maintenance is carried out automatically as part of the maintenance</span> +<span class="sd"> intervals (:py:obj:`MAINTENANCE_PERIOD`); no external process is required.</span> + +<span class="sd"> ``off``:</span> +<span class="sd"> Maintenance is switched off and must be carried out by an external process</span> +<span class="sd"> if required.</span> +<span class="sd"> """</span></div> + + + +<div class="viewcode-block" id="FaviconCacheStats"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheStats">[docs]</a> +<span class="nd">@dataclasses</span><span class="o">.</span><span class="n">dataclass</span> +<span class="k">class</span> <span class="nc">FaviconCacheStats</span><span class="p">:</span> +<span class="w"> </span><span class="sd">"""Dataclass wich provides information on the status of the cache."""</span> + + <span class="n">favicons</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span> + <span class="nb">bytes</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span> + <span class="n">domains</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span> + <span class="n">resolvers</span><span class="p">:</span> <span class="nb">int</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span> + + <span class="n">field_descr</span> <span class="o">=</span> <span class="p">(</span> + <span class="p">(</span><span class="s2">"favicons"</span><span class="p">,</span> <span class="s2">"number of favicons in cache"</span><span class="p">,</span> <span class="n">humanize_number</span><span class="p">),</span> + <span class="p">(</span><span class="s2">"bytes"</span><span class="p">,</span> <span class="s2">"total size (approx. bytes) of cache"</span><span class="p">,</span> <span class="n">humanize_bytes</span><span class="p">),</span> + <span class="p">(</span><span class="s2">"domains"</span><span class="p">,</span> <span class="s2">"total number of domains in cache"</span><span class="p">,</span> <span class="n">humanize_number</span><span class="p">),</span> + <span class="p">(</span><span class="s2">"resolvers"</span><span class="p">,</span> <span class="s2">"number of resolvers"</span><span class="p">,</span> <span class="nb">str</span><span class="p">),</span> + <span class="p">)</span> + + <span class="k">def</span> <span class="fm">__sub__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">)</span> <span class="o">-></span> <span class="n">FaviconCacheStats</span><span class="p">:</span> + <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="p">):</span> + <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"unsupported operand type(s) for +: '</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="si">}</span><span class="s2">' and '</span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">other</span><span class="p">)</span><span class="si">}</span><span class="s2">'"</span><span class="p">)</span> + <span class="n">kwargs</span> <span class="o">=</span> <span class="p">{}</span> + <span class="k">for</span> <span class="n">field</span><span class="p">,</span> <span class="n">_</span><span class="p">,</span> <span class="n">_</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">field_descr</span><span class="p">:</span> + <span class="n">self_val</span><span class="p">,</span> <span class="n">other_val</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">field</span><span class="p">),</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">field</span><span class="p">)</span> + <span class="k">if</span> <span class="kc">None</span> <span class="ow">in</span> <span class="p">(</span><span class="n">self_val</span><span class="p">,</span> <span class="n">other_val</span><span class="p">):</span> + <span class="k">continue</span> + <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">self_val</span><span class="p">,</span> <span class="nb">int</span><span class="p">):</span> + <span class="n">kwargs</span><span class="p">[</span><span class="n">field</span><span class="p">]</span> <span class="o">=</span> <span class="n">self_val</span> <span class="o">-</span> <span class="n">other_val</span> + <span class="k">else</span><span class="p">:</span> + <span class="n">kwargs</span><span class="p">[</span><span class="n">field</span><span class="p">]</span> <span class="o">=</span> <span class="n">self_val</span> + <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> + + <span class="k">def</span> <span class="nf">report</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">fmt</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"</span><span class="si">{descr}</span><span class="s2">: </span><span class="si">{val}</span><span class="se">\n</span><span class="s2">"</span><span class="p">):</span> + <span class="n">s</span> <span class="o">=</span> <span class="p">[]</span> + <span class="k">for</span> <span class="n">field</span><span class="p">,</span> <span class="n">descr</span><span class="p">,</span> <span class="n">cast</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">field_descr</span><span class="p">:</span> + <span class="n">val</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">field</span><span class="p">)</span> + <span class="k">if</span> <span class="n">val</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> + <span class="n">val</span> <span class="o">=</span> <span class="s2">"--"</span> + <span class="k">else</span><span class="p">:</span> + <span class="n">val</span> <span class="o">=</span> <span class="n">cast</span><span class="p">(</span><span class="n">val</span><span class="p">)</span> + <span class="n">s</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">fmt</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">descr</span><span class="o">=</span><span class="n">descr</span><span class="p">,</span> <span class="n">val</span><span class="o">=</span><span class="n">val</span><span class="p">))</span> + <span class="k">return</span> <span class="s2">""</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">s</span><span class="p">)</span></div> + + + +<div class="viewcode-block" id="FaviconCache"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCache">[docs]</a> +<span class="k">class</span> <span class="nc">FaviconCache</span><span class="p">(</span><span class="n">abc</span><span class="o">.</span><span class="n">ABC</span><span class="p">):</span> +<span class="w"> </span><span class="sd">"""Abstract base class for the implementation of a favicon cache."""</span> + + <span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span> + <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cfg</span><span class="p">:</span> <span class="n">FaviconCacheConfig</span><span class="p">):</span> +<span class="w"> </span><span class="sd">"""An instance of the favicon cache is build up from the configuration."""</span> + + <span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span> + <span class="k">def</span> <span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resolver</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">authority</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">tuple</span><span class="p">[</span><span class="kc">None</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">,</span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">str</span><span class="p">]:</span> +<span class="w"> </span><span class="sd">"""Returns ``None`` or the tuple of ``(data, mime)`` that has been</span> +<span class="sd"> registered in the cache. The ``None`` indicates that there was no entry</span> +<span class="sd"> in the cache."""</span> + +<div class="viewcode-block" id="FaviconCache.set"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCache.set">[docs]</a> + <span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span> + <span class="k">def</span> <span class="nf">set</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resolver</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">authority</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">mime</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="n">data</span><span class="p">:</span> <span class="nb">bytes</span> <span class="o">|</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> +<span class="w"> </span><span class="sd">"""Set data and mime-type in the cache. If data is None, the</span> +<span class="sd"> :py:obj:`FALLBACK_ICON` is registered. in the cache."""</span></div> + + +<div class="viewcode-block" id="FaviconCache.state"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCache.state">[docs]</a> + <span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span> + <span class="k">def</span> <span class="nf">state</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">FaviconCacheStats</span><span class="p">:</span> +<span class="w"> </span><span class="sd">"""Returns a :py:obj:`FaviconCacheStats` (key/values) with information</span> +<span class="sd"> on the state of the cache."""</span></div> + + +<div class="viewcode-block" id="FaviconCache.maintenance"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCache.maintenance">[docs]</a> + <span class="nd">@abc</span><span class="o">.</span><span class="n">abstractmethod</span> + <span class="k">def</span> <span class="nf">maintenance</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">force</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> +<span class="w"> </span><span class="sd">"""Performs maintenance on the cache"""</span></div> +</div> + + + +<div class="viewcode-block" id="FaviconCacheNull"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheNull">[docs]</a> +<span class="k">class</span> <span class="nc">FaviconCacheNull</span><span class="p">(</span><span class="n">FaviconCache</span><span class="p">):</span> +<span class="w"> </span><span class="sd">"""A dummy favicon cache that caches nothing / a fallback solution. The</span> +<span class="sd"> NullCache is used when more efficient caches such as the</span> +<span class="sd"> :py:obj:`FaviconCacheSQLite` cannot be used because, for example, the SQLite</span> +<span class="sd"> library is only available in an old version and does not meet the</span> +<span class="sd"> requirements."""</span> + + <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cfg</span><span class="p">:</span> <span class="n">FaviconCacheConfig</span><span class="p">):</span> + <span class="k">return</span> <span class="kc">None</span> + + <span class="k">def</span> <span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resolver</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">authority</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">tuple</span><span class="p">[</span><span class="kc">None</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">,</span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">str</span><span class="p">]:</span> + <span class="k">return</span> <span class="kc">None</span> + +<div class="viewcode-block" id="FaviconCacheNull.set"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheNull.set">[docs]</a> + <span class="k">def</span> <span class="nf">set</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resolver</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">authority</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">mime</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="n">data</span><span class="p">:</span> <span class="nb">bytes</span> <span class="o">|</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> + <span class="k">return</span> <span class="kc">False</span></div> + + +<div class="viewcode-block" id="FaviconCacheNull.state"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheNull.state">[docs]</a> + <span class="k">def</span> <span class="nf">state</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> + <span class="k">return</span> <span class="n">FaviconCacheStats</span><span class="p">(</span><span class="n">favicons</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span></div> + + +<div class="viewcode-block" id="FaviconCacheNull.maintenance"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheNull.maintenance">[docs]</a> + <span class="k">def</span> <span class="nf">maintenance</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">force</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> + <span class="k">pass</span></div> +</div> + + + +<div class="viewcode-block" id="FaviconCacheSQLite"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheSQLite">[docs]</a> +<span class="k">class</span> <span class="nc">FaviconCacheSQLite</span><span class="p">(</span><span class="n">sqlitedb</span><span class="o">.</span><span class="n">SQLiteAppl</span><span class="p">,</span> <span class="n">FaviconCache</span><span class="p">):</span> +<span class="w"> </span><span class="sd">"""Favicon cache that manages the favicon BLOBs in a SQLite DB. The DB</span> +<span class="sd"> model in the SQLite DB is implemented using the abstract class</span> +<span class="sd"> :py:obj:`sqlitedb.SQLiteAppl`.</span> + +<span class="sd"> The following configurations are required / supported:</span> + +<span class="sd"> - :py:obj:`FaviconCacheConfig.db_url`</span> +<span class="sd"> - :py:obj:`FaviconCacheConfig.HOLD_TIME`</span> +<span class="sd"> - :py:obj:`FaviconCacheConfig.LIMIT_TOTAL_BYTES`</span> +<span class="sd"> - :py:obj:`FaviconCacheConfig.BLOB_MAX_BYTES`</span> +<span class="sd"> - :py:obj:`MAINTENANCE_PERIOD`</span> +<span class="sd"> - :py:obj:`MAINTENANCE_MODE`</span> +<span class="sd"> """</span> + + <span class="n">DB_SCHEMA</span> <span class="o">=</span> <span class="mi">1</span> + + <span class="n">DDL_BLOBS</span> <span class="o">=</span> <span class="s2">"""</span><span class="se">\</span> +<span class="s2">CREATE TABLE IF NOT EXISTS blobs (</span> +<span class="s2"> sha256 TEXT,</span> +<span class="s2"> bytes_c INTEGER,</span> +<span class="s2"> mime TEXT NOT NULL,</span> +<span class="s2"> data BLOB NOT NULL,</span> +<span class="s2"> PRIMARY KEY (sha256))"""</span> + +<span class="w"> </span><span class="sd">"""Table to store BLOB objects by their sha256 hash values."""</span> + + <span class="n">DDL_BLOB_MAP</span> <span class="o">=</span> <span class="s2">"""</span><span class="se">\</span> +<span class="s2">CREATE TABLE IF NOT EXISTS blob_map (</span> +<span class="s2"> m_time INTEGER DEFAULT (strftime('</span><span class="si">%s</span><span class="s2">', 'now')), -- last modified (unix epoch) time in sec.</span> +<span class="s2"> sha256 TEXT,</span> +<span class="s2"> resolver TEXT,</span> +<span class="s2"> authority TEXT,</span> +<span class="s2"> PRIMARY KEY (resolver, authority))"""</span> + +<span class="w"> </span><span class="sd">"""Table to map from (resolver, authority) to sha256 hash values."""</span> + + <span class="n">DDL_CREATE_TABLES</span> <span class="o">=</span> <span class="p">{</span> + <span class="s2">"blobs"</span><span class="p">:</span> <span class="n">DDL_BLOBS</span><span class="p">,</span> + <span class="s2">"blob_map"</span><span class="p">:</span> <span class="n">DDL_BLOB_MAP</span><span class="p">,</span> + <span class="p">}</span> + + <span class="n">SQL_DROP_LEFTOVER_BLOBS</span> <span class="o">=</span> <span class="p">(</span> + <span class="s2">"DELETE FROM blobs WHERE sha256 IN ("</span> + <span class="s2">" SELECT b.sha256"</span> + <span class="s2">" FROM blobs b"</span> + <span class="s2">" LEFT JOIN blob_map bm"</span> + <span class="s2">" ON b.sha256 = bm.sha256"</span> + <span class="s2">" WHERE bm.sha256 IS NULL)"</span> + <span class="p">)</span> +<span class="w"> </span><span class="sd">"""Delete blobs.sha256 (BLOBs) no longer in blob_map.sha256."""</span> + + <span class="n">SQL_ITER_BLOBS_SHA256_BYTES_C</span> <span class="o">=</span> <span class="p">(</span> + <span class="s2">"SELECT b.sha256, b.bytes_c FROM blobs b"</span> + <span class="s2">" JOIN blob_map bm "</span> + <span class="s2">" ON b.sha256 = bm.sha256"</span> + <span class="s2">" ORDER BY bm.m_time ASC"</span> + <span class="p">)</span> + + <span class="n">SQL_INSERT_BLOBS</span> <span class="o">=</span> <span class="p">(</span> + <span class="s2">"INSERT INTO blobs (sha256, bytes_c, mime, data) VALUES (?, ?, ?, ?)"</span> + <span class="s2">" ON CONFLICT (sha256) DO NOTHING"</span> + <span class="p">)</span> <span class="c1"># fmt: skip</span> + + <span class="n">SQL_INSERT_BLOB_MAP</span> <span class="o">=</span> <span class="p">(</span> + <span class="s2">"INSERT INTO blob_map (sha256, resolver, authority) VALUES (?, ?, ?)"</span> + <span class="s2">" ON CONFLICT DO UPDATE "</span> + <span class="s2">" SET sha256=excluded.sha256, m_time=strftime('</span><span class="si">%s</span><span class="s2">', 'now')"</span> + <span class="p">)</span> + + <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cfg</span><span class="p">:</span> <span class="n">FaviconCacheConfig</span><span class="p">):</span> +<span class="w"> </span><span class="sd">"""An instance of the favicon cache is build up from the configuration."""</span> <span class="c1">#</span> + + <span class="k">if</span> <span class="n">cfg</span><span class="o">.</span><span class="n">db_url</span> <span class="o">==</span> <span class="s2">":memory:"</span><span class="p">:</span> + <span class="n">logger</span><span class="o">.</span><span class="n">critical</span><span class="p">(</span><span class="s2">"don't use SQLite DB in :memory: in production!!"</span><span class="p">)</span> + <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">cfg</span><span class="o">.</span><span class="n">db_url</span><span class="p">)</span> + <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span> <span class="o">=</span> <span class="n">cfg</span> + + <span class="k">def</span> <span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resolver</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">authority</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">tuple</span><span class="p">[</span><span class="kc">None</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">,</span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">str</span><span class="p">]:</span> + + <span class="n">sql</span> <span class="o">=</span> <span class="s2">"SELECT sha256 FROM blob_map WHERE resolver = ? AND authority = ?"</span> + <span class="n">res</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">DB</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">sql</span><span class="p">,</span> <span class="p">(</span><span class="n">resolver</span><span class="p">,</span> <span class="n">authority</span><span class="p">))</span><span class="o">.</span><span class="n">fetchone</span><span class="p">()</span> + <span class="k">if</span> <span class="n">res</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> + <span class="k">return</span> <span class="kc">None</span> + + <span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> + <span class="n">sha256</span> <span class="o">=</span> <span class="n">res</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> + <span class="k">if</span> <span class="n">sha256</span> <span class="o">==</span> <span class="n">FALLBACK_ICON</span><span class="p">:</span> + <span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">mime</span> + + <span class="n">sql</span> <span class="o">=</span> <span class="s2">"SELECT data, mime FROM blobs WHERE sha256 = ?"</span> + <span class="n">res</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">DB</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">sql</span><span class="p">,</span> <span class="p">(</span><span class="n">sha256</span><span class="p">,))</span><span class="o">.</span><span class="n">fetchone</span><span class="p">()</span> + <span class="k">if</span> <span class="n">res</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> + <span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="n">res</span> + <span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">mime</span> + +<div class="viewcode-block" id="FaviconCacheSQLite.set"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheSQLite.set">[docs]</a> + <span class="k">def</span> <span class="nf">set</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resolver</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">authority</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">mime</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="n">data</span><span class="p">:</span> <span class="nb">bytes</span> <span class="o">|</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> + + <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">MAINTENANCE_MODE</span> <span class="o">==</span> <span class="s2">"auto"</span> <span class="ow">and</span> <span class="nb">int</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">())</span> <span class="o">></span> <span class="bp">self</span><span class="o">.</span><span class="n">next_maintenance_time</span><span class="p">:</span> + <span class="c1"># Should automatic maintenance be moved to a new thread?</span> + <span class="bp">self</span><span class="o">.</span><span class="n">maintenance</span><span class="p">()</span> + + <span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">mime</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> + <span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span> + <span class="s2">"favicon resolver </span><span class="si">%s</span><span class="s2"> tries to cache mime-type None for authority </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> + <span class="n">resolver</span><span class="p">,</span> + <span class="n">authority</span><span class="p">,</span> + <span class="p">)</span> + <span class="k">return</span> <span class="kc">False</span> + + <span class="n">bytes_c</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">data</span> <span class="ow">or</span> <span class="sa">b</span><span class="s2">""</span><span class="p">)</span> + <span class="k">if</span> <span class="n">bytes_c</span> <span class="o">></span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">BLOB_MAX_BYTES</span><span class="p">:</span> + <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span> + <span class="s2">"favicon of resolver: </span><span class="si">%s</span><span class="s2"> / authority: </span><span class="si">%s</span><span class="s2"> to big to cache (bytes: </span><span class="si">%s</span><span class="s2">) "</span> <span class="o">%</span> <span class="p">(</span><span class="n">resolver</span><span class="p">,</span> <span class="n">authority</span><span class="p">,</span> <span class="n">bytes_c</span><span class="p">)</span> + <span class="p">)</span> + <span class="k">return</span> <span class="kc">False</span> + + <span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> + <span class="n">sha256</span> <span class="o">=</span> <span class="n">FALLBACK_ICON</span> + <span class="k">else</span><span class="p">:</span> + <span class="n">sha256</span> <span class="o">=</span> <span class="n">hashlib</span><span class="o">.</span><span class="n">sha256</span><span class="p">(</span><span class="n">data</span><span class="p">)</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">()</span> + + <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">connect</span><span class="p">()</span> <span class="k">as</span> <span class="n">conn</span><span class="p">:</span> + <span class="k">if</span> <span class="n">sha256</span> <span class="o">!=</span> <span class="n">FALLBACK_ICON</span><span class="p">:</span> + <span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">SQL_INSERT_BLOBS</span><span class="p">,</span> <span class="p">(</span><span class="n">sha256</span><span class="p">,</span> <span class="n">bytes_c</span><span class="p">,</span> <span class="n">mime</span><span class="p">,</span> <span class="n">data</span><span class="p">))</span> + <span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">SQL_INSERT_BLOB_MAP</span><span class="p">,</span> <span class="p">(</span><span class="n">sha256</span><span class="p">,</span> <span class="n">resolver</span><span class="p">,</span> <span class="n">authority</span><span class="p">))</span> + + <span class="k">return</span> <span class="kc">True</span></div> + + + <span class="nd">@property</span> + <span class="k">def</span> <span class="nf">next_maintenance_time</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">int</span><span class="p">:</span> +<span class="w"> </span><span class="sd">"""Returns (unix epoch) time of the next maintenance."""</span> + + <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">MAINTENANCE_PERIOD</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">properties</span><span class="o">.</span><span class="n">m_time</span><span class="p">(</span><span class="s2">"LAST_MAINTENANCE"</span><span class="p">)</span> + +<div class="viewcode-block" id="FaviconCacheSQLite.maintenance"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheSQLite.maintenance">[docs]</a> + <span class="k">def</span> <span class="nf">maintenance</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">force</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> + + <span class="c1"># Prevent parallel DB maintenance cycles from other DB connections</span> + <span class="c1"># (e.g. in multi thread or process environments).</span> + + <span class="k">if</span> <span class="ow">not</span> <span class="n">force</span> <span class="ow">and</span> <span class="nb">int</span><span class="p">(</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">())</span> <span class="o"><</span> <span class="bp">self</span><span class="o">.</span><span class="n">next_maintenance_time</span><span class="p">:</span> + <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"no maintenance required yet, next maintenance interval is in the future"</span><span class="p">)</span> + <span class="k">return</span> + <span class="bp">self</span><span class="o">.</span><span class="n">properties</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="s2">"LAST_MAINTENANCE"</span><span class="p">,</span> <span class="s2">""</span><span class="p">)</span> <span class="c1"># hint: this (also) sets the m_time of the property!</span> + + <span class="c1"># do maintenance tasks</span> + + <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">connect</span><span class="p">()</span> <span class="k">as</span> <span class="n">conn</span><span class="p">:</span> + + <span class="c1"># drop items not in HOLD time</span> + <span class="n">res</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span> + <span class="sa">f</span><span class="s2">"DELETE FROM blob_map"</span> + <span class="sa">f</span><span class="s2">" WHERE cast(m_time as integer) < cast(strftime('%s', 'now') as integer) - </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">HOLD_TIME</span><span class="si">}</span><span class="s2">"</span> + <span class="p">)</span> + <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"dropped </span><span class="si">%s</span><span class="s2"> obsolete blob_map items from db"</span><span class="p">,</span> <span class="n">res</span><span class="o">.</span><span class="n">rowcount</span><span class="p">)</span> + <span class="n">res</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">SQL_DROP_LEFTOVER_BLOBS</span><span class="p">)</span> + <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"dropped </span><span class="si">%s</span><span class="s2"> obsolete BLOBS from db"</span><span class="p">,</span> <span class="n">res</span><span class="o">.</span><span class="n">rowcount</span><span class="p">)</span> + + <span class="c1"># drop old items to be in LIMIT_TOTAL_BYTES</span> + <span class="n">total_bytes</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="s2">"SELECT SUM(bytes_c) FROM blobs"</span><span class="p">)</span><span class="o">.</span><span class="n">fetchone</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span> <span class="ow">or</span> <span class="mi">0</span> + <span class="k">if</span> <span class="n">total_bytes</span> <span class="o">></span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">LIMIT_TOTAL_BYTES</span><span class="p">:</span> + + <span class="n">x</span> <span class="o">=</span> <span class="n">total_bytes</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span><span class="o">.</span><span class="n">LIMIT_TOTAL_BYTES</span> + <span class="n">c</span> <span class="o">=</span> <span class="mi">0</span> + <span class="n">sha_list</span> <span class="o">=</span> <span class="p">[]</span> + <span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">SQL_ITER_BLOBS_SHA256_BYTES_C</span><span class="p">):</span> + <span class="n">sha256</span><span class="p">,</span> <span class="n">bytes_c</span> <span class="o">=</span> <span class="n">row</span> + <span class="n">sha_list</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">sha256</span><span class="p">)</span> + <span class="n">c</span> <span class="o">+=</span> <span class="n">bytes_c</span> + <span class="k">if</span> <span class="n">c</span> <span class="o">></span> <span class="n">x</span><span class="p">:</span> + <span class="k">break</span> + <span class="k">if</span> <span class="n">sha_list</span><span class="p">:</span> + <span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="s2">"DELETE FROM blobs WHERE sha256 IN ('</span><span class="si">%s</span><span class="s2">')"</span> <span class="o">%</span> <span class="s2">"','"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">sha_list</span><span class="p">))</span> + <span class="n">conn</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="s2">"DELETE FROM blob_map WHERE sha256 IN ('</span><span class="si">%s</span><span class="s2">')"</span> <span class="o">%</span> <span class="s2">"','"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">sha_list</span><span class="p">))</span> + <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"dropped </span><span class="si">%s</span><span class="s2"> blobs with total size of </span><span class="si">%s</span><span class="s2"> bytes"</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">sha_list</span><span class="p">),</span> <span class="n">c</span><span class="p">)</span></div> + + + <span class="k">def</span> <span class="nf">_query_val</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span> + <span class="n">val</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">DB</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">sql</span><span class="p">)</span><span class="o">.</span><span class="n">fetchone</span><span class="p">()</span> + <span class="k">if</span> <span class="n">val</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> + <span class="n">val</span> <span class="o">=</span> <span class="n">val</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> + <span class="k">if</span> <span class="n">val</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> + <span class="n">val</span> <span class="o">=</span> <span class="n">default</span> + <span class="k">return</span> <span class="n">val</span> + +<div class="viewcode-block" id="FaviconCacheSQLite.state"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheSQLite.state">[docs]</a> + <span class="k">def</span> <span class="nf">state</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">FaviconCacheStats</span><span class="p">:</span> + <span class="k">return</span> <span class="n">FaviconCacheStats</span><span class="p">(</span> + <span class="n">favicons</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_query_val</span><span class="p">(</span><span class="s2">"SELECT count(*) FROM blobs"</span><span class="p">,</span> <span class="mi">0</span><span class="p">),</span> + <span class="nb">bytes</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_query_val</span><span class="p">(</span><span class="s2">"SELECT SUM(bytes_c) FROM blobs"</span><span class="p">,</span> <span class="mi">0</span><span class="p">),</span> + <span class="n">domains</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_query_val</span><span class="p">(</span><span class="s2">"SELECT count(*) FROM (SELECT authority FROM blob_map GROUP BY authority)"</span><span class="p">,</span> <span class="mi">0</span><span class="p">),</span> + <span class="n">resolvers</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_query_val</span><span class="p">(</span><span class="s2">"SELECT count(*) FROM (SELECT resolver FROM blob_map GROUP BY resolver)"</span><span class="p">,</span> <span class="mi">0</span><span class="p">),</span> + <span class="p">)</span></div> +</div> + + + +<div class="viewcode-block" id="FaviconCacheMEM"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheMEM">[docs]</a> +<span class="k">class</span> <span class="nc">FaviconCacheMEM</span><span class="p">(</span><span class="n">FaviconCache</span><span class="p">):</span> +<span class="w"> </span><span class="sd">"""Favicon cache in process' memory. Its just a POC that stores the</span> +<span class="sd"> favicons in the memory of the process.</span> + +<span class="sd"> .. attention::</span> + +<span class="sd"> Don't use it in production, it will blow up your memory!!</span> + +<span class="sd"> """</span> + + <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cfg</span><span class="p">):</span> + + <span class="bp">self</span><span class="o">.</span><span class="n">cfg</span> <span class="o">=</span> <span class="n">cfg</span> + <span class="bp">self</span><span class="o">.</span><span class="n">_data</span> <span class="o">=</span> <span class="p">{}</span> + <span class="bp">self</span><span class="o">.</span><span class="n">_sha_mime</span> <span class="o">=</span> <span class="p">{}</span> + + <span class="k">def</span> <span class="fm">__call__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resolver</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">authority</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">bytes</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">]:</span> + + <span class="n">sha</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_sha_mime</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">resolver</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="n">authority</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">))</span> + <span class="k">if</span> <span class="n">sha</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> + <span class="k">return</span> <span class="kc">None</span> + <span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">sha</span><span class="p">)</span> + <span class="k">if</span> <span class="n">data</span> <span class="o">==</span> <span class="n">FALLBACK_ICON</span><span class="p">:</span> + <span class="n">data</span> <span class="o">=</span> <span class="kc">None</span> + <span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">mime</span> + +<div class="viewcode-block" id="FaviconCacheMEM.set"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheMEM.set">[docs]</a> + <span class="k">def</span> <span class="nf">set</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">resolver</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">authority</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">mime</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span><span class="p">,</span> <span class="n">data</span><span class="p">:</span> <span class="nb">bytes</span> <span class="o">|</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-></span> <span class="nb">bool</span><span class="p">:</span> + + <span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> + <span class="n">data</span> <span class="o">=</span> <span class="n">FALLBACK_ICON</span> + <span class="n">mime</span> <span class="o">=</span> <span class="kc">None</span> + + <span class="k">elif</span> <span class="n">mime</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> + <span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span> + <span class="s2">"favicon resolver </span><span class="si">%s</span><span class="s2"> tries to cache mime-type None for authority </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> + <span class="n">resolver</span><span class="p">,</span> + <span class="n">authority</span><span class="p">,</span> + <span class="p">)</span> + <span class="k">return</span> <span class="kc">False</span> + + <span class="n">digest</span> <span class="o">=</span> <span class="n">hashlib</span><span class="o">.</span><span class="n">sha256</span><span class="p">(</span><span class="n">data</span><span class="p">)</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">()</span> + <span class="bp">self</span><span class="o">.</span><span class="n">_data</span><span class="p">[</span><span class="n">digest</span><span class="p">]</span> <span class="o">=</span> <span class="n">data</span> + <span class="bp">self</span><span class="o">.</span><span class="n">_sha_mime</span><span class="p">[</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">resolver</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="n">authority</span><span class="si">}</span><span class="s2">"</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="n">digest</span><span class="p">,</span> <span class="n">mime</span><span class="p">)</span> + <span class="k">return</span> <span class="kc">True</span></div> + + +<div class="viewcode-block" id="FaviconCacheMEM.state"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheMEM.state">[docs]</a> + <span class="k">def</span> <span class="nf">state</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span> + <span class="k">return</span> <span class="n">FaviconCacheStats</span><span class="p">(</span><span class="n">favicons</span><span class="o">=</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_data</span><span class="o">.</span><span class="n">keys</span><span class="p">()))</span></div> + + +<div class="viewcode-block" id="FaviconCacheMEM.maintenance"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.cache.FaviconCacheMEM.maintenance">[docs]</a> + <span class="k">def</span> <span class="nf">maintenance</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">force</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span> + <span class="k">pass</span></div> +</div> + +</pre></div> + + <div class="clearer"></div> + </div> + </div> + </div> + <span id="sidebar-top"></span> + <div class="sphinxsidebar" role="navigation" aria-label="Main"> + <div class="sphinxsidebarwrapper"> + + + <p class="logo"><a href="../../../index.html"> + <img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/> + </a></p> + + +<h3><a href="../../../index.html">Table of Contents</a></h3> +<ul> +<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li> +</ul> + + <h3>Project Links</h3> + <ul> + <li><a href="https://github.com/searxng/searxng/tree/master">Source</a> + + <li><a href="https://github.com/searxng/searxng/wiki">Wiki</a> + + <li><a href="https://searx.space">Public instances</a> + + <li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a> + </ul><h3>Navigation</h3> +<ul> + <li><a href="../../../index.html">Overview</a> + <ul> + <li><a href="../../index.html">Module code</a> + + + </ul> + </li> + </ul> + </li> +</ul> +<search id="searchbox" style="display: none" role="search"> + <h3 id="searchlabel">Quick search</h3> + <div class="searchformwrapper"> + <form class="search" action="../../../search.html" method="get"> + <input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/> + <input type="submit" value="Go" /> + </form> + </div> +</search> +<script>document.getElementById('searchbox').style.display = "block"</script> + </div> + </div> + <div class="clearer"></div> + </div> + <div class="footer" role="contentinfo"> + © Copyright SearXNG team. + </div> + </body> +</html>
\ No newline at end of file diff --git a/_modules/searx/favicons/config.html b/_modules/searx/favicons/config.html new file mode 100644 index 000000000..2524b7315 --- /dev/null +++ b/_modules/searx/favicons/config.html @@ -0,0 +1,175 @@ +<!DOCTYPE html> + +<html lang="en" data-content_root="../../../"> + <head> + <meta charset="utf-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <title>searx.favicons.config — SearXNG Documentation (2025.1.6+6dab7fe78)</title> + <link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=4f649999" /> + <link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" /> + <script src="../../../_static/documentation_options.js?v=ef740023"></script> + <script src="../../../_static/doctools.js?v=9a2dae69"></script> + <script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script> + <script data-project="searxng" data-version="2025.1.6+6dab7fe78" src="../../../_static/describe_version.js?v=fa7f30d0"></script> + <link rel="index" title="Index" href="../../../genindex.html" /> + <link rel="search" title="Search" href="../../../search.html" /> + </head><body> + <div class="related" role="navigation" aria-label="Related"> + <h3>Navigation</h3> + <ul> + <li class="right" style="margin-right: 10px"> + <a href="../../../genindex.html" title="General Index" + accesskey="I">index</a></li> + <li class="right" > + <a href="../../../py-modindex.html" title="Python Module Index" + >modules</a> |</li> + <li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2025.1.6+6dab7fe78)</a> »</li> + <li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li> + <li class="nav-item nav-item-this"><a href="">searx.favicons.config</a></li> + </ul> + </div> + + <div class="document"> + <div class="documentwrapper"> + <div class="bodywrapper"> + <div class="body" role="main"> + + <h1>Source code for searx.favicons.config</h1><div class="highlight"><pre> +<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span> +<span class="c1"># pylint: disable=missing-module-docstring</span> + +<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">annotations</span> + +<span class="kn">import</span> <span class="nn">pathlib</span> +<span class="kn">import</span> <span class="nn">msgspec</span> + +<span class="kn">from</span> <span class="nn">.cache</span> <span class="kn">import</span> <span class="n">FaviconCacheConfig</span> +<span class="kn">from</span> <span class="nn">.proxy</span> <span class="kn">import</span> <span class="n">FaviconProxyConfig</span> + +<span class="n">CONFIG_SCHEMA</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span> +<span class="sd">"""Version of the configuration schema."""</span> + +<span class="n">TOML_CACHE_CFG</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="s2">"FaviconConfig"</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span> +<span class="sd">"""Cache config objects by TOML's filename."""</span> + +<span class="n">DEFAULT_CFG_TOML_PATH</span> <span class="o">=</span> <span class="n">pathlib</span><span class="o">.</span><span class="n">Path</span><span class="p">(</span><span class="vm">__file__</span><span class="p">)</span><span class="o">.</span><span class="n">parent</span> <span class="o">/</span> <span class="s2">"favicons.toml"</span> + + +<div class="viewcode-block" id="FaviconConfig"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.config.FaviconConfig">[docs]</a> +<span class="k">class</span> <span class="nc">FaviconConfig</span><span class="p">(</span><span class="n">msgspec</span><span class="o">.</span><span class="n">Struct</span><span class="p">):</span> <span class="c1"># pylint: disable=too-few-public-methods</span> +<span class="w"> </span><span class="sd">"""The class aggregates configurations of the favicon tools"""</span> + + <span class="n">cfg_schema</span><span class="p">:</span> <span class="nb">int</span> +<span class="w"> </span><span class="sd">"""Config's schema version. The specification of the version of the schema</span> +<span class="sd"> is mandatory, currently only version :py:obj:`CONFIG_SCHEMA` is supported.</span> +<span class="sd"> By specifying a version, it is possible to ensure downward compatibility in</span> +<span class="sd"> the event of future changes to the configuration schema"""</span> + + <span class="n">cache</span><span class="p">:</span> <span class="n">FaviconCacheConfig</span> <span class="o">=</span> <span class="n">msgspec</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="n">default_factory</span><span class="o">=</span><span class="n">FaviconCacheConfig</span><span class="p">)</span> +<span class="w"> </span><span class="sd">"""Setup of the :py:obj:`.cache.FaviconCacheConfig`."""</span> + + <span class="n">proxy</span><span class="p">:</span> <span class="n">FaviconProxyConfig</span> <span class="o">=</span> <span class="n">msgspec</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="n">default_factory</span><span class="o">=</span><span class="n">FaviconProxyConfig</span><span class="p">)</span> +<span class="w"> </span><span class="sd">"""Setup of the :py:obj:`.proxy.FaviconProxyConfig`."""</span> + +<div class="viewcode-block" id="FaviconConfig.from_toml_file"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.config.FaviconConfig.from_toml_file">[docs]</a> + <span class="nd">@classmethod</span> + <span class="k">def</span> <span class="nf">from_toml_file</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">cfg_file</span><span class="p">:</span> <span class="n">pathlib</span><span class="o">.</span><span class="n">Path</span><span class="p">,</span> <span class="n">use_cache</span><span class="p">:</span> <span class="nb">bool</span><span class="p">)</span> <span class="o">-></span> <span class="s2">"FaviconConfig"</span><span class="p">:</span> +<span class="w"> </span><span class="sd">"""Create a config object from a TOML file, the ``use_cache`` argument</span> +<span class="sd"> specifies whether a cache should be used.</span> +<span class="sd"> """</span> + + <span class="n">cached</span> <span class="o">=</span> <span class="n">TOML_CACHE_CFG</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">cfg_file</span><span class="p">))</span> + <span class="k">if</span> <span class="n">use_cache</span> <span class="ow">and</span> <span class="n">cached</span><span class="p">:</span> + <span class="k">return</span> <span class="n">cached</span> + + <span class="k">with</span> <span class="n">cfg_file</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="s2">"rb"</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span> + <span class="n">data</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">()</span> + + <span class="n">cfg</span> <span class="o">=</span> <span class="n">msgspec</span><span class="o">.</span><span class="n">toml</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="n">_FaviconConfig</span><span class="p">)</span> + <span class="n">schema</span> <span class="o">=</span> <span class="n">cfg</span><span class="o">.</span><span class="n">favicons</span><span class="o">.</span><span class="n">cfg_schema</span> + <span class="k">if</span> <span class="n">schema</span> <span class="o">!=</span> <span class="n">CONFIG_SCHEMA</span><span class="p">:</span> + <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span> + <span class="sa">f</span><span class="s2">"config schema version </span><span class="si">{</span><span class="n">CONFIG_SCHEMA</span><span class="si">}</span><span class="s2"> is needed, version </span><span class="si">{</span><span class="n">schema</span><span class="si">}</span><span class="s2"> is given in </span><span class="si">{</span><span class="n">cfg_file</span><span class="si">}</span><span class="s2">"</span> + <span class="p">)</span> + + <span class="n">cfg</span> <span class="o">=</span> <span class="n">cfg</span><span class="o">.</span><span class="n">favicons</span> + <span class="k">if</span> <span class="n">use_cache</span> <span class="ow">and</span> <span class="n">cached</span><span class="p">:</span> + <span class="n">TOML_CACHE_CFG</span><span class="p">[</span><span class="nb">str</span><span class="p">(</span><span class="n">cfg_file</span><span class="o">.</span><span class="n">resolve</span><span class="p">())]</span> <span class="o">=</span> <span class="n">cfg</span> + + <span class="k">return</span> <span class="n">cfg</span></div> +</div> + + + +<span class="k">class</span> <span class="nc">_FaviconConfig</span><span class="p">(</span><span class="n">msgspec</span><span class="o">.</span><span class="n">Struct</span><span class="p">):</span> <span class="c1"># pylint: disable=too-few-public-methods</span> + <span class="c1"># wrapper struct for root object "favicons."</span> + <span class="n">favicons</span><span class="p">:</span> <span class="n">FaviconConfig</span> +</pre></div> + + <div class="clearer"></div> + </div> + </div> + </div> + <span id="sidebar-top"></span> + <div class="sphinxsidebar" role="navigation" aria-label="Main"> + <div class="sphinxsidebarwrapper"> + + + <p class="logo"><a href="../../../index.html"> + <img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/> + </a></p> + + +<h3><a href="../../../index.html">Table of Contents</a></h3> +<ul> +<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li> +</ul> + + <h3>Project Links</h3> + <ul> + <li><a href="https://github.com/searxng/searxng/tree/master">Source</a> + + <li><a href="https://github.com/searxng/searxng/wiki">Wiki</a> + + <li><a href="https://searx.space">Public instances</a> + + <li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a> + </ul><h3>Navigation</h3> +<ul> + <li><a href="../../../index.html">Overview</a> + <ul> + <li><a href="../../index.html">Module code</a> + + + </ul> + </li> + </ul> + </li> +</ul> +<search id="searchbox" style="display: none" role="search"> + <h3 id="searchlabel">Quick search</h3> + <div class="searchformwrapper"> + <form class="search" action="../../../search.html" method="get"> + <input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/> + <input type="submit" value="Go" /> + </form> + </div> +</search> +<script>document.getElementById('searchbox').style.display = "block"</script> + </div> + </div> + <div class="clearer"></div> + </div> + <div class="footer" role="contentinfo"> + © Copyright SearXNG team. + </div> + </body> +</html>
\ No newline at end of file diff --git a/_modules/searx/favicons/proxy.html b/_modules/searx/favicons/proxy.html new file mode 100644 index 000000000..8ab92e6b5 --- /dev/null +++ b/_modules/searx/favicons/proxy.html @@ -0,0 +1,362 @@ +<!DOCTYPE html> + +<html lang="en" data-content_root="../../../"> + <head> + <meta charset="utf-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <title>searx.favicons.proxy — SearXNG Documentation (2025.1.6+6dab7fe78)</title> + <link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=4f649999" /> + <link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" /> + <script src="../../../_static/documentation_options.js?v=ef740023"></script> + <script src="../../../_static/doctools.js?v=9a2dae69"></script> + <script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script> + <script data-project="searxng" data-version="2025.1.6+6dab7fe78" src="../../../_static/describe_version.js?v=fa7f30d0"></script> + <link rel="index" title="Index" href="../../../genindex.html" /> + <link rel="search" title="Search" href="../../../search.html" /> + </head><body> + <div class="related" role="navigation" aria-label="Related"> + <h3>Navigation</h3> + <ul> + <li class="right" style="margin-right: 10px"> + <a href="../../../genindex.html" title="General Index" + accesskey="I">index</a></li> + <li class="right" > + <a href="../../../py-modindex.html" title="Python Module Index" + >modules</a> |</li> + <li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2025.1.6+6dab7fe78)</a> »</li> + <li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li> + <li class="nav-item nav-item-this"><a href="">searx.favicons.proxy</a></li> + </ul> + </div> + + <div class="document"> + <div class="documentwrapper"> + <div class="bodywrapper"> + <div class="body" role="main"> + + <h1>Source code for searx.favicons.proxy</h1><div class="highlight"><pre> +<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span> +<span class="sd">"""Implementations for a favicon proxy"""</span> + +<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">annotations</span> + +<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Callable</span> + +<span class="kn">import</span> <span class="nn">importlib</span> +<span class="kn">import</span> <span class="nn">base64</span> +<span class="kn">import</span> <span class="nn">pathlib</span> +<span class="kn">import</span> <span class="nn">urllib.parse</span> + +<span class="kn">import</span> <span class="nn">flask</span> +<span class="kn">from</span> <span class="nn">httpx</span> <span class="kn">import</span> <span class="n">HTTPError</span> +<span class="kn">import</span> <span class="nn">msgspec</span> + +<span class="kn">from</span> <span class="nn">searx</span> <span class="kn">import</span> <span class="n">get_setting</span> + +<span class="kn">from</span> <span class="nn">searx.webutils</span> <span class="kn">import</span> <span class="n">new_hmac</span><span class="p">,</span> <span class="n">is_hmac_of</span> +<span class="kn">from</span> <span class="nn">searx.exceptions</span> <span class="kn">import</span> <span class="n">SearxEngineResponseException</span> + +<span class="kn">from</span> <span class="nn">.resolvers</span> <span class="kn">import</span> <span class="n">DEFAULT_RESOLVER_MAP</span> +<span class="kn">from</span> <span class="nn">.</span> <span class="kn">import</span> <span class="n">cache</span> + +<span class="n">DEFAULT_FAVICON_URL</span> <span class="o">=</span> <span class="p">{}</span> +<span class="n">CFG</span><span class="p">:</span> <span class="n">FaviconProxyConfig</span> <span class="o">=</span> <span class="kc">None</span> <span class="c1"># type: ignore</span> + + +<span class="k">def</span> <span class="nf">init</span><span class="p">(</span><span class="n">cfg</span><span class="p">:</span> <span class="n">FaviconProxyConfig</span><span class="p">):</span> + <span class="k">global</span> <span class="n">CFG</span> <span class="c1"># pylint: disable=global-statement</span> + <span class="n">CFG</span> <span class="o">=</span> <span class="n">cfg</span> + + +<span class="k">def</span> <span class="nf">_initial_resolver_map</span><span class="p">():</span> + <span class="n">d</span> <span class="o">=</span> <span class="p">{}</span> + <span class="n">name</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">get_setting</span><span class="p">(</span><span class="s2">"search.favicon_resolver"</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> <span class="c1"># type: ignore</span> + <span class="k">if</span> <span class="n">name</span><span class="p">:</span> + <span class="n">func</span> <span class="o">=</span> <span class="n">DEFAULT_RESOLVER_MAP</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">name</span><span class="p">)</span> + <span class="k">if</span> <span class="n">func</span><span class="p">:</span> + <span class="n">d</span> <span class="o">=</span> <span class="p">{</span><span class="n">name</span><span class="p">:</span> <span class="sa">f</span><span class="s2">"searx.favicons.resolvers.</span><span class="si">{</span><span class="n">func</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="s2">"</span><span class="p">}</span> + <span class="k">return</span> <span class="n">d</span> + + +<div class="viewcode-block" id="FaviconProxyConfig"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.FaviconProxyConfig">[docs]</a> +<span class="k">class</span> <span class="nc">FaviconProxyConfig</span><span class="p">(</span><span class="n">msgspec</span><span class="o">.</span><span class="n">Struct</span><span class="p">):</span> +<span class="w"> </span><span class="sd">"""Configuration of the favicon proxy."""</span> + + <span class="n">max_age</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">60</span> <span class="o">*</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">7</span> <span class="c1"># seven days</span> +<span class="w"> </span><span class="sd">"""HTTP header Cache-Control_ ``max-age``</span> + +<span class="sd"> .. _Cache-Control: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control</span> +<span class="sd"> """</span> + + <span class="n">secret_key</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">get_setting</span><span class="p">(</span><span class="s2">"server.secret_key"</span><span class="p">)</span> <span class="c1"># type: ignore</span> +<span class="w"> </span><span class="sd">"""By default, the value from :ref:`server.secret_key <settings server>`</span> +<span class="sd"> setting is used."""</span> + + <span class="n">resolver_timeout</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="n">get_setting</span><span class="p">(</span><span class="s2">"outgoing.request_timeout"</span><span class="p">)</span> <span class="c1"># type: ignore</span> +<span class="w"> </span><span class="sd">"""Timeout which the resolvers should not exceed, is usually passed to the</span> +<span class="sd"> outgoing request of the resolver. By default, the value from</span> +<span class="sd"> :ref:`outgoing.request_timeout <settings outgoing>` setting is used."""</span> + + <span class="n">resolver_map</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">msgspec</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="n">default_factory</span><span class="o">=</span><span class="n">_initial_resolver_map</span><span class="p">)</span> +<span class="w"> </span><span class="sd">"""The resolver_map is a key / value dictionary where the key is the name of</span> +<span class="sd"> the resolver and the value is the fully qualifying name (fqn) of resolver's</span> +<span class="sd"> function (the callable). The resolvers from the python module</span> +<span class="sd"> :py:obj:`searx.favicons.resolver` are available by default."""</span> + +<div class="viewcode-block" id="FaviconProxyConfig.get_resolver"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.FaviconProxyConfig.get_resolver">[docs]</a> + <span class="k">def</span> <span class="nf">get_resolver</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="n">Callable</span> <span class="o">|</span> <span class="kc">None</span><span class="p">:</span> +<span class="w"> </span><span class="sd">"""Returns the callable object (function) of the resolver with the</span> +<span class="sd"> ``name``. If no resolver is registered for the ``name``, ``None`` is</span> +<span class="sd"> returned.</span> +<span class="sd"> """</span> + <span class="n">fqn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">resolver_map</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">name</span><span class="p">)</span> + <span class="k">if</span> <span class="n">fqn</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> + <span class="k">return</span> <span class="kc">None</span> + <span class="n">mod_name</span><span class="p">,</span> <span class="n">_</span><span class="p">,</span> <span class="n">func_name</span> <span class="o">=</span> <span class="n">fqn</span><span class="o">.</span><span class="n">rpartition</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)</span> + <span class="n">mod</span> <span class="o">=</span> <span class="n">importlib</span><span class="o">.</span><span class="n">import_module</span><span class="p">(</span><span class="n">mod_name</span><span class="p">)</span> + <span class="n">func</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">mod</span><span class="p">,</span> <span class="n">func_name</span><span class="p">)</span> + <span class="k">if</span> <span class="n">func</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> + <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"resolver </span><span class="si">{</span><span class="n">fqn</span><span class="si">}</span><span class="s2"> is not implemented"</span><span class="p">)</span> + <span class="k">return</span> <span class="n">func</span></div> + + + <span class="n">favicon_path</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">get_setting</span><span class="p">(</span><span class="s2">"ui.static_path"</span><span class="p">)</span> <span class="o">+</span> <span class="s2">"/themes/</span><span class="si">{theme}</span><span class="s2">/img/empty_favicon.svg"</span> <span class="c1"># type: ignore</span> + <span class="n">favicon_mime_type</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"image/svg+xml"</span> + +<div class="viewcode-block" id="FaviconProxyConfig.favicon"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.FaviconProxyConfig.favicon">[docs]</a> + <span class="k">def</span> <span class="nf">favicon</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">replacements</span><span class="p">):</span> +<span class="w"> </span><span class="sd">"""Returns pathname and mimetype of the default favicon."""</span> + <span class="k">return</span> <span class="p">(</span> + <span class="n">pathlib</span><span class="o">.</span><span class="n">Path</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">favicon_path</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="n">replacements</span><span class="p">)),</span> + <span class="bp">self</span><span class="o">.</span><span class="n">favicon_mime_type</span><span class="p">,</span> + <span class="p">)</span></div> + + +<div class="viewcode-block" id="FaviconProxyConfig.favicon_data_url"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.FaviconProxyConfig.favicon_data_url">[docs]</a> + <span class="k">def</span> <span class="nf">favicon_data_url</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">replacements</span><span class="p">):</span> +<span class="w"> </span><span class="sd">"""Returns data image URL of the default favicon."""</span> + + <span class="n">cache_key</span> <span class="o">=</span> <span class="s2">", "</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">x</span><span class="si">}</span><span class="s2">:</span><span class="si">{</span><span class="n">replacements</span><span class="p">[</span><span class="n">x</span><span class="p">]</span><span class="si">}</span><span class="s2">"</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">sorted</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="n">replacements</span><span class="o">.</span><span class="n">keys</span><span class="p">()),</span> <span class="n">key</span><span class="o">=</span><span class="nb">str</span><span class="p">))</span> + <span class="n">data_url</span> <span class="o">=</span> <span class="n">DEFAULT_FAVICON_URL</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">cache_key</span><span class="p">)</span> + <span class="k">if</span> <span class="n">data_url</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> + <span class="k">return</span> <span class="n">data_url</span> + + <span class="n">fav</span><span class="p">,</span> <span class="n">mimetype</span> <span class="o">=</span> <span class="n">CFG</span><span class="o">.</span><span class="n">favicon</span><span class="p">(</span><span class="o">**</span><span class="n">replacements</span><span class="p">)</span> + <span class="c1"># hint: encoding utf-8 limits favicons to be a SVG image</span> + <span class="k">with</span> <span class="n">fav</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="s2">"r"</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">"utf-8"</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span> + <span class="n">data_url</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">()</span> + + <span class="n">data_url</span> <span class="o">=</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">quote</span><span class="p">(</span><span class="n">data_url</span><span class="p">)</span> + <span class="n">data_url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"data:</span><span class="si">{</span><span class="n">mimetype</span><span class="si">}</span><span class="s2">;utf8,</span><span class="si">{</span><span class="n">data_url</span><span class="si">}</span><span class="s2">"</span> + <span class="n">DEFAULT_FAVICON_URL</span><span class="p">[</span><span class="n">cache_key</span><span class="p">]</span> <span class="o">=</span> <span class="n">data_url</span> + <span class="k">return</span> <span class="n">data_url</span></div> +</div> + + + +<div class="viewcode-block" id="favicon_proxy"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.favicon_proxy">[docs]</a> +<span class="k">def</span> <span class="nf">favicon_proxy</span><span class="p">():</span> +<span class="w"> </span><span class="sd">"""REST API of SearXNG's favicon proxy service</span> + +<span class="sd"> ::</span> + +<span class="sd"> /favicon_proxy?authority=<...>&h=<...></span> + +<span class="sd"> ``authority``:</span> +<span class="sd"> Domain name :rfc:`3986` / see :py:obj:`favicon_url`</span> + +<span class="sd"> ``h``:</span> +<span class="sd"> HMAC :rfc:`2104`, build up from the :ref:`server.secret_key <settings</span> +<span class="sd"> server>` setting.</span> + +<span class="sd"> """</span> + <span class="n">authority</span> <span class="o">=</span> <span class="n">flask</span><span class="o">.</span><span class="n">request</span><span class="o">.</span><span class="n">args</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'authority'</span><span class="p">)</span> + + <span class="c1"># malformed request or RFC 3986 authority</span> + <span class="k">if</span> <span class="ow">not</span> <span class="n">authority</span> <span class="ow">or</span> <span class="s2">"/"</span> <span class="ow">in</span> <span class="n">authority</span><span class="p">:</span> + <span class="k">return</span> <span class="s1">''</span><span class="p">,</span> <span class="mi">400</span> + + <span class="c1"># malformed request / does not have authorisation</span> + <span class="k">if</span> <span class="ow">not</span> <span class="n">is_hmac_of</span><span class="p">(</span> + <span class="n">CFG</span><span class="o">.</span><span class="n">secret_key</span><span class="p">,</span> + <span class="n">authority</span><span class="o">.</span><span class="n">encode</span><span class="p">(),</span> + <span class="n">flask</span><span class="o">.</span><span class="n">request</span><span class="o">.</span><span class="n">args</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'h'</span><span class="p">,</span> <span class="s1">''</span><span class="p">),</span> + <span class="p">):</span> + <span class="k">return</span> <span class="s1">''</span><span class="p">,</span> <span class="mi">400</span> + + <span class="n">resolver</span> <span class="o">=</span> <span class="n">flask</span><span class="o">.</span><span class="n">request</span><span class="o">.</span><span class="n">preferences</span><span class="o">.</span><span class="n">get_value</span><span class="p">(</span><span class="s1">'favicon_resolver'</span><span class="p">)</span> <span class="c1"># type: ignore</span> + <span class="c1"># if resolver is empty or not valid, just return HTTP 400.</span> + <span class="k">if</span> <span class="ow">not</span> <span class="n">resolver</span> <span class="ow">or</span> <span class="n">resolver</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">CFG</span><span class="o">.</span><span class="n">resolver_map</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span> + <span class="k">return</span> <span class="s2">""</span><span class="p">,</span> <span class="mi">400</span> + + <span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="n">search_favicon</span><span class="p">(</span><span class="n">resolver</span><span class="p">,</span> <span class="n">authority</span><span class="p">)</span> + + <span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">mime</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> + <span class="n">resp</span> <span class="o">=</span> <span class="n">flask</span><span class="o">.</span><span class="n">Response</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">mimetype</span><span class="o">=</span><span class="n">mime</span><span class="p">)</span> <span class="c1"># type: ignore</span> + <span class="n">resp</span><span class="o">.</span><span class="n">headers</span><span class="p">[</span><span class="s1">'Cache-Control'</span><span class="p">]</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"max-age=</span><span class="si">{</span><span class="n">CFG</span><span class="o">.</span><span class="n">max_age</span><span class="si">}</span><span class="s2">"</span> + <span class="k">return</span> <span class="n">resp</span> + + <span class="c1"># return default favicon from static path</span> + <span class="n">theme</span> <span class="o">=</span> <span class="n">flask</span><span class="o">.</span><span class="n">request</span><span class="o">.</span><span class="n">preferences</span><span class="o">.</span><span class="n">get_value</span><span class="p">(</span><span class="s2">"theme"</span><span class="p">)</span> <span class="c1"># type: ignore</span> + <span class="n">fav</span><span class="p">,</span> <span class="n">mimetype</span> <span class="o">=</span> <span class="n">CFG</span><span class="o">.</span><span class="n">favicon</span><span class="p">(</span><span class="n">theme</span><span class="o">=</span><span class="n">theme</span><span class="p">)</span> + <span class="k">return</span> <span class="n">flask</span><span class="o">.</span><span class="n">send_from_directory</span><span class="p">(</span><span class="n">fav</span><span class="o">.</span><span class="n">parent</span><span class="p">,</span> <span class="n">fav</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">mimetype</span><span class="o">=</span><span class="n">mimetype</span><span class="p">)</span></div> + + + +<div class="viewcode-block" id="search_favicon"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.search_favicon">[docs]</a> +<span class="k">def</span> <span class="nf">search_favicon</span><span class="p">(</span><span class="n">resolver</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">authority</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">tuple</span><span class="p">[</span><span class="kc">None</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">,</span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">str</span><span class="p">]:</span> +<span class="w"> </span><span class="sd">"""Sends the request to the favicon resolver and returns a tuple for the</span> +<span class="sd"> favicon. The tuple consists of ``(data, mime)``, if the resolver has not</span> +<span class="sd"> determined a favicon, both values are ``None``.</span> + +<span class="sd"> ``data``:</span> +<span class="sd"> Binary data of the favicon.</span> + +<span class="sd"> ``mime``:</span> +<span class="sd"> Mime type of the favicon.</span> + +<span class="sd"> """</span> + + <span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> + + <span class="n">func</span> <span class="o">=</span> <span class="n">CFG</span><span class="o">.</span><span class="n">get_resolver</span><span class="p">(</span><span class="n">resolver</span><span class="p">)</span> + <span class="k">if</span> <span class="n">func</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> + <span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">mime</span> + + <span class="c1"># to avoid superfluous requests to the resolver, first look in the cache</span> + <span class="n">data_mime</span> <span class="o">=</span> <span class="n">cache</span><span class="o">.</span><span class="n">CACHE</span><span class="p">(</span><span class="n">resolver</span><span class="p">,</span> <span class="n">authority</span><span class="p">)</span> + <span class="k">if</span> <span class="n">data_mime</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> + <span class="k">return</span> <span class="n">data_mime</span> + + <span class="k">try</span><span class="p">:</span> + <span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="n">func</span><span class="p">(</span><span class="n">authority</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="n">CFG</span><span class="o">.</span><span class="n">resolver_timeout</span><span class="p">)</span> + <span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="n">mime</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span> + <span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> + + <span class="k">except</span> <span class="p">(</span><span class="n">HTTPError</span><span class="p">,</span> <span class="n">SearxEngineResponseException</span><span class="p">):</span> + <span class="k">pass</span> + + <span class="n">cache</span><span class="o">.</span><span class="n">CACHE</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="n">resolver</span><span class="p">,</span> <span class="n">authority</span><span class="p">,</span> <span class="n">mime</span><span class="p">,</span> <span class="n">data</span><span class="p">)</span> + <span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">mime</span></div> + + + +<div class="viewcode-block" id="favicon_url"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.favicon_url">[docs]</a> +<span class="k">def</span> <span class="nf">favicon_url</span><span class="p">(</span><span class="n">authority</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span> +<span class="w"> </span><span class="sd">"""Function to generate the image URL used for favicons in SearXNG's result</span> +<span class="sd"> lists. The ``authority`` argument (aka netloc / :rfc:`3986`) is usually a</span> +<span class="sd"> (sub-) domain name. This function is used in the HTML (jinja) templates.</span> + +<span class="sd"> .. code:: html</span> + +<span class="sd"> <div class="favicon"></span> +<span class="sd"> <img src="{{ favicon_url(result.parsed_url.netloc) }}"></span> +<span class="sd"> </div></span> + +<span class="sd"> The returned URL is a route to :py:obj:`favicon_proxy` REST API.</span> + +<span class="sd"> If the favicon is already in the cache, the returned URL is a `data URL`_</span> +<span class="sd"> (something like ``data:image/png;base64,...``). By generating a data url from</span> +<span class="sd"> the :py:obj:`.cache.FaviconCache`, additional HTTP roundtripps via the</span> +<span class="sd"> :py:obj:`favicon_proxy` are saved. However, it must also be borne in mind</span> +<span class="sd"> that data urls are not cached in the client (web browser).</span> + +<span class="sd"> .. _data URL: https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URLs</span> + +<span class="sd"> """</span> + + <span class="n">resolver</span> <span class="o">=</span> <span class="n">flask</span><span class="o">.</span><span class="n">request</span><span class="o">.</span><span class="n">preferences</span><span class="o">.</span><span class="n">get_value</span><span class="p">(</span><span class="s1">'favicon_resolver'</span><span class="p">)</span> <span class="c1"># type: ignore</span> + <span class="c1"># if resolver is empty or not valid, just return nothing.</span> + <span class="k">if</span> <span class="ow">not</span> <span class="n">resolver</span> <span class="ow">or</span> <span class="n">resolver</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">CFG</span><span class="o">.</span><span class="n">resolver_map</span><span class="o">.</span><span class="n">keys</span><span class="p">():</span> + <span class="k">return</span> <span class="s2">""</span> + + <span class="n">data_mime</span> <span class="o">=</span> <span class="n">cache</span><span class="o">.</span><span class="n">CACHE</span><span class="p">(</span><span class="n">resolver</span><span class="p">,</span> <span class="n">authority</span><span class="p">)</span> + + <span class="k">if</span> <span class="n">data_mime</span> <span class="o">==</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">):</span> + <span class="c1"># we have already checked, the resolver does not have a favicon</span> + <span class="n">theme</span> <span class="o">=</span> <span class="n">flask</span><span class="o">.</span><span class="n">request</span><span class="o">.</span><span class="n">preferences</span><span class="o">.</span><span class="n">get_value</span><span class="p">(</span><span class="s2">"theme"</span><span class="p">)</span> <span class="c1"># type: ignore</span> + <span class="k">return</span> <span class="n">CFG</span><span class="o">.</span><span class="n">favicon_data_url</span><span class="p">(</span><span class="n">theme</span><span class="o">=</span><span class="n">theme</span><span class="p">)</span> + + <span class="k">if</span> <span class="n">data_mime</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span> + <span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="n">data_mime</span> + <span class="k">return</span> <span class="sa">f</span><span class="s2">"data:</span><span class="si">{</span><span class="n">mime</span><span class="si">}</span><span class="s2">;base64,</span><span class="si">{</span><span class="nb">str</span><span class="p">(</span><span class="n">base64</span><span class="o">.</span><span class="n">b64encode</span><span class="p">(</span><span class="n">data</span><span class="p">),</span><span class="w"> </span><span class="s1">'utf-8'</span><span class="p">)</span><span class="si">}</span><span class="s2">"</span> <span class="c1"># type: ignore</span> + + <span class="n">h</span> <span class="o">=</span> <span class="n">new_hmac</span><span class="p">(</span><span class="n">CFG</span><span class="o">.</span><span class="n">secret_key</span><span class="p">,</span> <span class="n">authority</span><span class="o">.</span><span class="n">encode</span><span class="p">())</span> + <span class="n">proxy_url</span> <span class="o">=</span> <span class="n">flask</span><span class="o">.</span><span class="n">url_for</span><span class="p">(</span><span class="s1">'favicon_proxy'</span><span class="p">)</span> + <span class="n">query</span> <span class="o">=</span> <span class="n">urllib</span><span class="o">.</span><span class="n">parse</span><span class="o">.</span><span class="n">urlencode</span><span class="p">({</span><span class="s2">"authority"</span><span class="p">:</span> <span class="n">authority</span><span class="p">,</span> <span class="s2">"h"</span><span class="p">:</span> <span class="n">h</span><span class="p">})</span> + <span class="k">return</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">proxy_url</span><span class="si">}</span><span class="s2">?</span><span class="si">{</span><span class="n">query</span><span class="si">}</span><span class="s2">"</span></div> + +</pre></div> + + <div class="clearer"></div> + </div> + </div> + </div> + <span id="sidebar-top"></span> + <div class="sphinxsidebar" role="navigation" aria-label="Main"> + <div class="sphinxsidebarwrapper"> + + + <p class="logo"><a href="../../../index.html"> + <img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/> + </a></p> + + +<h3><a href="../../../index.html">Table of Contents</a></h3> +<ul> +<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li> +</ul> + + <h3>Project Links</h3> + <ul> + <li><a href="https://github.com/searxng/searxng/tree/master">Source</a> + + <li><a href="https://github.com/searxng/searxng/wiki">Wiki</a> + + <li><a href="https://searx.space">Public instances</a> + + <li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a> + </ul><h3>Navigation</h3> +<ul> + <li><a href="../../../index.html">Overview</a> + <ul> + <li><a href="../../index.html">Module code</a> + + + </ul> + </li> + </ul> + </li> +</ul> +<search id="searchbox" style="display: none" role="search"> + <h3 id="searchlabel">Quick search</h3> + <div class="searchformwrapper"> + <form class="search" action="../../../search.html" method="get"> + <input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/> + <input type="submit" value="Go" /> + </form> + </div> +</search> +<script>document.getElementById('searchbox').style.display = "block"</script> + </div> + </div> + <div class="clearer"></div> + </div> + <div class="footer" role="contentinfo"> + © Copyright SearXNG team. + </div> + </body> +</html>
\ No newline at end of file diff --git a/_modules/searx/favicons/resolvers.html b/_modules/searx/favicons/resolvers.html new file mode 100644 index 000000000..4ab53f417 --- /dev/null +++ b/_modules/searx/favicons/resolvers.html @@ -0,0 +1,216 @@ +<!DOCTYPE html> + +<html lang="en" data-content_root="../../../"> + <head> + <meta charset="utf-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <title>searx.favicons.resolvers — SearXNG Documentation (2025.1.6+6dab7fe78)</title> + <link rel="stylesheet" type="text/css" href="../../../_static/pygments.css?v=4f649999" /> + <link rel="stylesheet" type="text/css" href="../../../_static/searxng.css?v=52e4ff28" /> + <script src="../../../_static/documentation_options.js?v=ef740023"></script> + <script src="../../../_static/doctools.js?v=9a2dae69"></script> + <script src="../../../_static/sphinx_highlight.js?v=dc90522c"></script> + <script data-project="searxng" data-version="2025.1.6+6dab7fe78" src="../../../_static/describe_version.js?v=fa7f30d0"></script> + <link rel="index" title="Index" href="../../../genindex.html" /> + <link rel="search" title="Search" href="../../../search.html" /> + </head><body> + <div class="related" role="navigation" aria-label="Related"> + <h3>Navigation</h3> + <ul> + <li class="right" style="margin-right: 10px"> + <a href="../../../genindex.html" title="General Index" + accesskey="I">index</a></li> + <li class="right" > + <a href="../../../py-modindex.html" title="Python Module Index" + >modules</a> |</li> + <li class="nav-item nav-item-0"><a href="../../../index.html">SearXNG Documentation (2025.1.6+6dab7fe78)</a> »</li> + <li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> »</li> + <li class="nav-item nav-item-this"><a href="">searx.favicons.resolvers</a></li> + </ul> + </div> + + <div class="document"> + <div class="documentwrapper"> + <div class="bodywrapper"> + <div class="body" role="main"> + + <h1>Source code for searx.favicons.resolvers</h1><div class="highlight"><pre> +<span></span><span class="c1"># SPDX-License-Identifier: AGPL-3.0-or-later</span> +<span class="sd">"""Implementations of the favicon *resolvers* that are available in the favicon</span> +<span class="sd">proxy by default. A *resolver* is a function that obtains the favicon from an</span> +<span class="sd">external source. The *resolver* function receives two arguments (``domain,</span> +<span class="sd">timeout``) and returns a tuple ``(data, mime)``.</span> + +<span class="sd">"""</span> + +<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">annotations</span> + +<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"DEFAULT_RESOLVER_MAP"</span><span class="p">,</span> <span class="s2">"allesedv"</span><span class="p">,</span> <span class="s2">"duckduckgo"</span><span class="p">,</span> <span class="s2">"google"</span><span class="p">,</span> <span class="s2">"yandex"</span><span class="p">]</span> + +<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Callable</span> +<span class="kn">from</span> <span class="nn">searx</span> <span class="kn">import</span> <span class="n">network</span> +<span class="kn">from</span> <span class="nn">searx</span> <span class="kn">import</span> <span class="n">logger</span> + +<span class="n">DEFAULT_RESOLVER_MAP</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Callable</span><span class="p">]</span> +<span class="n">logger</span> <span class="o">=</span> <span class="n">logger</span><span class="o">.</span><span class="n">getChild</span><span class="p">(</span><span class="s1">'favicons.resolvers'</span><span class="p">)</span> + + +<span class="k">def</span> <span class="nf">_req_args</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">):</span> + <span class="c1"># add the request arguments from the searx.network</span> + <span class="n">d</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"raise_for_httperror"</span><span class="p">:</span> <span class="kc">False</span><span class="p">}</span> + <span class="n">d</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">kwargs</span><span class="p">)</span> + <span class="k">return</span> <span class="n">d</span> + + +<div class="viewcode-block" id="allesedv"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.resolvers.allesedv">[docs]</a> +<span class="k">def</span> <span class="nf">allesedv</span><span class="p">(</span><span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">timeout</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-></span> <span class="nb">tuple</span><span class="p">[</span><span class="kc">None</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">,</span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">str</span><span class="p">]:</span> +<span class="w"> </span><span class="sd">"""Favicon Resolver from allesedv.com / https://favicon.allesedv.com/"""</span> + <span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> + <span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"https://f1.allesedv.com/32/</span><span class="si">{</span><span class="n">domain</span><span class="si">}</span><span class="s2">"</span> + <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"fetch favicon from: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span> + + <span class="c1"># will just return a 200 regardless of the favicon existing or not</span> + <span class="c1"># sometimes will be correct size, sometimes not</span> + <span class="n">response</span> <span class="o">=</span> <span class="n">network</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="o">**</span><span class="n">_req_args</span><span class="p">(</span><span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">))</span> + <span class="k">if</span> <span class="n">response</span> <span class="ow">and</span> <span class="n">response</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">200</span><span class="p">:</span> + <span class="n">mime</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">headers</span><span class="p">[</span><span class="s1">'Content-Type'</span><span class="p">]</span> + <span class="k">if</span> <span class="n">mime</span> <span class="o">!=</span> <span class="s1">'image/gif'</span><span class="p">:</span> + <span class="n">data</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">content</span> + <span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">mime</span></div> + + + +<div class="viewcode-block" id="duckduckgo"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.resolvers.duckduckgo">[docs]</a> +<span class="k">def</span> <span class="nf">duckduckgo</span><span class="p">(</span><span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">timeout</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-></span> <span class="nb">tuple</span><span class="p">[</span><span class="kc">None</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">,</span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">str</span><span class="p">]:</span> +<span class="w"> </span><span class="sd">"""Favicon Resolver from duckduckgo.com / https://blog.jim-nielsen.com/2021/displaying-favicons-for-any-domain/"""</span> + <span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> + <span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"https://icons.duckduckgo.com/ip2/</span><span class="si">{</span><span class="n">domain</span><span class="si">}</span><span class="s2">.ico"</span> + <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"fetch favicon from: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span> + + <span class="c1"># will return a 404 if the favicon does not exist and a 200 if it does,</span> + <span class="n">response</span> <span class="o">=</span> <span class="n">network</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="o">**</span><span class="n">_req_args</span><span class="p">(</span><span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">))</span> + <span class="k">if</span> <span class="n">response</span> <span class="ow">and</span> <span class="n">response</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">200</span><span class="p">:</span> + <span class="c1"># api will respond with a 32x32 png image</span> + <span class="n">mime</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">headers</span><span class="p">[</span><span class="s1">'Content-Type'</span><span class="p">]</span> + <span class="n">data</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">content</span> + <span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">mime</span></div> + + + +<div class="viewcode-block" id="google"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.resolvers.google">[docs]</a> +<span class="k">def</span> <span class="nf">google</span><span class="p">(</span><span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">timeout</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-></span> <span class="nb">tuple</span><span class="p">[</span><span class="kc">None</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">,</span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">str</span><span class="p">]:</span> +<span class="w"> </span><span class="sd">"""Favicon Resolver from google.com"""</span> + <span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> + + <span class="c1"># URL https://www.google.com/s2/favicons?sz=32&domain={domain}" will be</span> + <span class="c1"># redirected (HTTP 301 Moved Permanently) to t1.gstatic.com/faviconV2:</span> + <span class="n">url</span> <span class="o">=</span> <span class="p">(</span> + <span class="sa">f</span><span class="s2">"https://t1.gstatic.com/faviconV2?client=SOCIAL&type=FAVICON&fallback_opts=TYPE,SIZE,URL"</span> + <span class="sa">f</span><span class="s2">"&url=https://</span><span class="si">{</span><span class="n">domain</span><span class="si">}</span><span class="s2">&size=32"</span> + <span class="p">)</span> + <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"fetch favicon from: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span> + + <span class="c1"># will return a 404 if the favicon does not exist and a 200 if it does,</span> + <span class="n">response</span> <span class="o">=</span> <span class="n">network</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="o">**</span><span class="n">_req_args</span><span class="p">(</span><span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">))</span> + <span class="k">if</span> <span class="n">response</span> <span class="ow">and</span> <span class="n">response</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">200</span><span class="p">:</span> + <span class="c1"># api will respond with a 32x32 png image</span> + <span class="n">mime</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">headers</span><span class="p">[</span><span class="s1">'Content-Type'</span><span class="p">]</span> + <span class="n">data</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">content</span> + <span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">mime</span></div> + + + +<div class="viewcode-block" id="yandex"> +<a class="viewcode-back" href="../../../src/searx.favicons.html#searx.favicons.resolvers.yandex">[docs]</a> +<span class="k">def</span> <span class="nf">yandex</span><span class="p">(</span><span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">timeout</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-></span> <span class="nb">tuple</span><span class="p">[</span><span class="kc">None</span> <span class="o">|</span> <span class="nb">bytes</span><span class="p">,</span> <span class="kc">None</span> <span class="o">|</span> <span class="nb">str</span><span class="p">]:</span> +<span class="w"> </span><span class="sd">"""Favicon Resolver from yandex.com"""</span> + <span class="n">data</span><span class="p">,</span> <span class="n">mime</span> <span class="o">=</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span> + <span class="n">url</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">"https://favicon.yandex.net/favicon/</span><span class="si">{</span><span class="n">domain</span><span class="si">}</span><span class="s2">"</span> + <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">"fetch favicon from: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span> + + <span class="c1"># api will respond with a 16x16 png image, if it doesn't exist, it will be a</span> + <span class="c1"># 1x1 png image (70 bytes)</span> + <span class="n">response</span> <span class="o">=</span> <span class="n">network</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="o">**</span><span class="n">_req_args</span><span class="p">(</span><span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">))</span> + <span class="k">if</span> <span class="n">response</span> <span class="ow">and</span> <span class="n">response</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="mi">200</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">content</span><span class="p">)</span> <span class="o">></span> <span class="mi">70</span><span class="p">:</span> + <span class="n">mime</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">headers</span><span class="p">[</span><span class="s1">'Content-Type'</span><span class="p">]</span> + <span class="n">data</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">content</span> + <span class="k">return</span> <span class="n">data</span><span class="p">,</span> <span class="n">mime</span></div> + + + +<span class="n">DEFAULT_RESOLVER_MAP</span> <span class="o">=</span> <span class="p">{</span> + <span class="s2">"allesedv"</span><span class="p">:</span> <span class="n">allesedv</span><span class="p">,</span> + <span class="s2">"duckduckgo"</span><span class="p">:</span> <span class="n">duckduckgo</span><span class="p">,</span> + <span class="s2">"google"</span><span class="p">:</span> <span class="n">google</span><span class="p">,</span> + <span class="s2">"yandex"</span><span class="p">:</span> <span class="n">yandex</span><span class="p">,</span> +<span class="p">}</span> +</pre></div> + + <div class="clearer"></div> + </div> + </div> + </div> + <span id="sidebar-top"></span> + <div class="sphinxsidebar" role="navigation" aria-label="Main"> + <div class="sphinxsidebarwrapper"> + + + <p class="logo"><a href="../../../index.html"> + <img class="logo" src="../../../_static/searxng-wordmark.svg" alt="Logo of SearXNG"/> + </a></p> + + +<h3><a href="../../../index.html">Table of Contents</a></h3> +<ul> +<li class="toctree-l1"><a class="reference internal" href="../../../user/index.html">User information</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../own-instance.html">Why use a private instance?</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../admin/index.html">Administrator documentation</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../dev/index.html">Developer documentation</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../utils/index.html">DevOps tooling box</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../../src/index.html">Source-Code</a></li> +</ul> + + <h3>Project Links</h3> + <ul> + <li><a href="https://github.com/searxng/searxng/tree/master">Source</a> + + <li><a href="https://github.com/searxng/searxng/wiki">Wiki</a> + + <li><a href="https://searx.space">Public instances</a> + + <li><a href="https://github.com/searxng/searxng/issues">Issue Tracker</a> + </ul><h3>Navigation</h3> +<ul> + <li><a href="../../../index.html">Overview</a> + <ul> + <li><a href="../../index.html">Module code</a> + + + </ul> + </li> + </ul> + </li> +</ul> +<search id="searchbox" style="display: none" role="search"> + <h3 id="searchlabel">Quick search</h3> + <div class="searchformwrapper"> + <form class="search" action="../../../search.html" method="get"> + <input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/> + <input type="submit" value="Go" /> + </form> + </div> +</search> +<script>document.getElementById('searchbox').style.display = "block"</script> + </div> + </div> + <div class="clearer"></div> + </div> + <div class="footer" role="contentinfo"> + © Copyright SearXNG team. + </div> + </body> +</html>
\ No newline at end of file |