diff options
| author | Markus Heiser <markus.heiser@darmarIT.de> | 2022-07-30 13:45:12 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2022-07-30 13:45:12 +0200 |
| commit | 645c2a2ca1cf40ab9429366ac70c69d1ac8fd004 (patch) | |
| tree | 943270e4998815d360713e21615082293300c877 /docs | |
| parent | 0cf5310933ee4f6bff4a941f7296686bd671671f (diff) | |
| parent | e5323b8aa285d61bf011bbe7ca8f02818f19ed79 (diff) | |
| download | searxng-645c2a2ca1cf40ab9429366ac70c69d1ac8fd004.tar.gz searxng-645c2a2ca1cf40ab9429366ac70c69d1ac8fd004.zip | |
Merge pull request #1332 from return42/searxng-install
Upgrade installation scripts and documentation
Diffstat (limited to 'docs')
32 files changed, 897 insertions, 1489 deletions
diff --git a/docs/admin/arch_public.dot b/docs/admin/arch_public.dot index c4ee5f3c1..526fb53da 100644 --- a/docs/admin/arch_public.dot +++ b/docs/admin/arch_public.dot @@ -1,33 +1,30 @@ digraph G { - node [style=filled, shape=box, fillcolor="#ffffcc", fontname="Sans"]; + node [style=filled, shape=box, fillcolor="#ffffcc", fontname=Sans]; edge [fontname="Sans"]; - browser [label="Browser", shape=Mdiamond]; - rp [label="Reverse Proxy", href="https://docs.searxng.org/utils/filtron.sh.html#public-reverse-proxy"]; - filtron [label="Filtron", href="https://docs.searxng.org/utils/filtron.sh.html"]; - morty [label="Morty", href="https://docs.searxng.org/utils/morty.sh.html"]; - static [label="Static files", href="url to configure static files"]; - uwsgi [label="uwsgi", href="https://docs.searxng.org/utils/searx.sh.html"] - searx1 [label="Searx #1"]; - searx2 [label="Searx #2"]; - searx3 [label="Searx #3"]; - searx4 [label="Searx #4"]; + browser [label="browser", shape=tab, fillcolor=aliceblue]; + rp [label="reverse proxy"]; + static [label="static files", shape=folder, href="url to configure static files", fillcolor=lightgray]; + uwsgi [label="uwsgi", shape=parallelogram href="https://docs.searxng.org/utils/searx.sh.html"] + redis [label="redis DB", shape=cylinder]; + searxng1 [label="SearXNG #1", fontcolor=blue3]; + searxng2 [label="SearXNG #2", fontcolor=blue3]; + searxng3 [label="SearXNG #3", fontcolor=blue3]; + searxng4 [label="SearXNG #4", fontcolor=blue3]; browser -> rp [label="HTTPS"] - subgraph cluster_searx { - label = "Searx instance" fontname="Sans"; + subgraph cluster_searxng { + label = "SearXNG instance" fontname=Sans; bgcolor="#fafafa"; { rank=same; static rp }; - rp -> morty [label="optional: images and HTML pages proxy"]; - rp -> static [label="optional: reverse proxy serves directly static files"]; - rp -> filtron [label="HTTP"]; - filtron -> uwsgi [label="HTTP"]; - uwsgi -> searx1; - uwsgi -> searx2; - uwsgi -> searx3; - uwsgi -> searx4; + rp -> static [label="optional: reverse proxy serves static files", fillcolor=slategray, fontcolor=slategray]; + rp -> uwsgi [label="http:// (tcp) or unix:// (socket)"]; + uwsgi -> searxng1 -> redis; + uwsgi -> searxng2 -> redis; + uwsgi -> searxng3 -> redis; + uwsgi -> searxng4 -> redis; } } diff --git a/docs/admin/architecture.rst b/docs/admin/architecture.rst index db99c9f55..d0d40715d 100644 --- a/docs/admin/architecture.rst +++ b/docs/admin/architecture.rst @@ -8,17 +8,19 @@ Architecture - Reverse Proxy: :ref:`Apache <apache searxng site>` & :ref:`nginx <nginx searxng site>` - - Filtron: :ref:`searxng filtron` - - Morty: :ref:`searxng morty` - uWSGI: :ref:`searxng uwsgi` - SearXNG: :ref:`installation basic` Herein you will find some hints and suggestions about typical architectures of SearXNG infrastructures. -We start with a contribution from :pull-searx:`@dalf <1776#issuecomment-567917320>`. -It shows a *reference* setup for public SearXNG instances which can build up and -maintained by the scripts from our :ref:`toolboxing`. +.. _architecture uWSGI: + +uWSGI Setup +=========== + +We start with a *reference* setup for public SearXNG instances which can be build +up and maintained by the scripts from our :ref:`toolboxing`. .. _arch public: @@ -26,3 +28,11 @@ maintained by the scripts from our :ref:`toolboxing`. :alt: arch_public.dot Reference architecture of a public SearXNG setup. + +The reference installation activates ``server.limiter``, ``server.image_proxy`` +and ``ui.static_use_hash`` (:origin:`/etc/searxng/settings.yml +<utils/templates/etc/searxng/settings.yml>`) + +.. literalinclude:: ../../utils/templates/etc/searxng/settings.yml + :language: yaml + :end-before: # preferences: diff --git a/docs/admin/buildhosts.rst b/docs/admin/buildhosts.rst index e23327ba8..6926469ef 100644 --- a/docs/admin/buildhosts.rst +++ b/docs/admin/buildhosts.rst @@ -15,19 +15,19 @@ Buildhosts :backlinks: entry To get best results from build, its recommend to install additional packages -on build hosts (see :ref:`searx.sh`).:: +on build hosts (see :ref:`searxng.sh`).:: - sudo -H ./utils/searx.sh install buildhost + sudo -H ./utils/searxng.sh install buildhost This will install packages needed by searx: -.. kernel-include:: $DOCS_BUILD/includes/searx.rst +.. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START distro-packages :end-before: END distro-packages and packages needed to build docuemtation and run tests: -.. kernel-include:: $DOCS_BUILD/includes/searx.rst +.. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START build-packages :end-before: END build-packages diff --git a/docs/admin/engines/nosql-engines.rst b/docs/admin/engines/nosql-engines.rst index a50b9c367..68fd0e8bc 100644 --- a/docs/admin/engines/nosql-engines.rst +++ b/docs/admin/engines/nosql-engines.rst @@ -42,11 +42,11 @@ Extra Dependencies For using :ref:`engine redis_server` or :ref:`engine mongodb` you need to install additional packages in Python's Virtual Environment of your SearXNG -instance. To switch into the environment (:ref:`searx-src`) you can use -:ref:`searx.sh`:: +instance. To switch into the environment (:ref:`searxng-src`) you can use +:ref:`searxng.sh`:: - $ sudo utils/searx.sh shell - (searx-pyenv)$ pip install ... + $ sudo utils/searxng.sh instance cmd bash + (searxng-pyenv)$ pip install ... .. _engine redis_server: diff --git a/docs/admin/engines/settings.rst b/docs/admin/engines/settings.rst index 71ad29b43..0b4b984d7 100644 --- a/docs/admin/engines/settings.rst +++ b/docs/admin/engines/settings.rst @@ -207,10 +207,14 @@ Global Settings ``secret_key`` : ``$SEARXNG_SECRET`` Used for cryptography purpose. +.. _limiter: + ``limiter`` : Rate limit the number of request on the instance, block some bots. The :ref:`limiter plugin` requires a :ref:`settings redis` database. +.. _image_proxy: + ``image_proxy`` : Allow your instance of SearXNG of being able to proxy images. Uses memory space. @@ -225,9 +229,13 @@ Global Settings ``ui:`` ------- +.. _cache busting: + https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control#caching_static_assets_with_cache_busting + .. code:: yaml ui: + static_use_hash: false default_locale: "" query_in_title: false infinite_scroll: false @@ -236,6 +244,11 @@ Global Settings theme_args: simple_style: auto +.. _static_use_hash: + +``static_use_hash`` : + Enables `cache busting`_ of static files. + ``default_locale`` : SearXNG interface language. If blank, the locale is detected by using the browser language. If it doesn't work, or you are deploying a language diff --git a/docs/admin/engines/sql-engines.rst b/docs/admin/engines/sql-engines.rst index d91383214..6b6a4cb68 100644 --- a/docs/admin/engines/sql-engines.rst +++ b/docs/admin/engines/sql-engines.rst @@ -98,11 +98,11 @@ Extra Dependencies For using :ref:`engine postgresql` or :ref:`engine mysql_server` you need to install additional packages in Python's Virtual Environment of your SearXNG -instance. To switch into the environment (:ref:`searx-src`) you can use -:ref:`searx.sh`:: +instance. To switch into the environment (:ref:`searxng-src`) you can use +:ref:`searxng.sh`:: - $ sudo utils/searx.sh shell - (searx-pyenv)$ pip install ... + $ sudo utils/searxng.sh instance cmd bash + (searxng-pyenv)$ pip install ... .. _engine postgresql: diff --git a/docs/admin/filtron.rst b/docs/admin/filtron.rst deleted file mode 100644 index 60d7cbeba..000000000 --- a/docs/admin/filtron.rst +++ /dev/null @@ -1,193 +0,0 @@ - -.. _searxng filtron: - -========================== -How to protect an instance -========================== - -.. tip:: - - To protect your instance a installation of filtron (as described here) is no - longer needed, alternatively activate the :ref:`limiter plugin` in your - ``settings.yml``. Note that the :ref:`limiter plugin` requires a :ref:`Redis - <settings redis>` database. - - -.. sidebar:: further reading - - - :ref:`filtron.sh` - - :ref:`nginx searxng site` - -.. _filtron: https://github.com/searxng/filtron - -SearXNG depends on external search services. To avoid the abuse of these services -it is advised to limit the number of requests processed by SearXNG. - -An application firewall, filtron_ solves exactly this problem. Filtron is just -a middleware between your web server (nginx, apache, ...) and searx, we describe -such infrastructures in chapter: :ref:`architecture`. - - -filtron & go -============ - -.. _Go: https://golang.org/ -.. _filtron README: https://github.com/searxng/filtron/blob/master/README.md - -Filtron needs Go_ installed. If Go_ is preinstalled, filtron_ is simply -installed by ``go get`` package management (see `filtron README`_). If you use -filtron as middleware, a more isolated setup is recommended. To simplify such -an installation and the maintenance of, use our script :ref:`filtron.sh`. - -.. _Sample configuration of filtron: - -Sample configuration of filtron -=============================== - -.. sidebar:: Tooling box - - - :origin:`/etc/filtron/rules.json <utils/templates/etc/filtron/rules.json>` - -An example configuration can be find below. This configuration limits the access -of: - -- scripts or applications (roboagent limit) -- webcrawlers (botlimit) -- IPs which send too many requests (IP limit) -- too many json, csv, etc. requests (rss/json limit) -- the same UserAgent of if too many requests (useragent limit) - -.. code:: json - - [ - { - "name": "search request", - "filters": [ - "Param:q", - "Path=^(/|/search)$" - ], - "interval": "<time-interval-in-sec (int)>", - "limit": "<max-request-number-in-interval (int)>", - "subrules": [ - { - "name": "missing Accept-Language", - "filters": ["!Header:Accept-Language"], - "limit": "<max-request-number-in-interval (int)>", - "stop": true, - "actions": [ - {"name":"log"}, - {"name": "block", - "params": {"message": "Rate limit exceeded"}} - ] - }, - { - "name": "suspiciously Connection=close header", - "filters": ["Header:Connection=close"], - "limit": "<max-request-number-in-interval (int)>", - "stop": true, - "actions": [ - {"name":"log"}, - {"name": "block", - "params": {"message": "Rate limit exceeded"}} - ] - }, - { - "name": "IP limit", - "interval": "<time-interval-in-sec (int)>", - "limit": "<max-request-number-in-interval (int)>", - "stop": true, - "aggregations": [ - "Header:X-Forwarded-For" - ], - "actions": [ - { "name": "log"}, - { "name": "block", - "params": { - "message": "Rate limit exceeded" - } - } - ] - }, - { - "name": "rss/json limit", - "filters": [ - "Param:format=(csv|json|rss)" - ], - "interval": "<time-interval-in-sec (int)>", - "limit": "<max-request-number-in-interval (int)>", - "stop": true, - "actions": [ - { "name": "log"}, - { "name": "block", - "params": { - "message": "Rate limit exceeded" - } - } - ] - }, - { - "name": "useragent limit", - "interval": "<time-interval-in-sec (int)>", - "limit": "<max-request-number-in-interval (int)>", - "aggregations": [ - "Header:User-Agent" - ], - "actions": [ - { "name": "log"}, - { "name": "block", - "params": { - "message": "Rate limit exceeded" - } - } - ] - } - ] - } - ] - - -.. _filtron route request: - -Route request through filtron -============================= - -.. sidebar:: further reading - - - :ref:`filtron.sh overview` - - :ref:`installation nginx` - - :ref:`installation apache` - -Filtron can be started using the following command: - -.. code:: sh - - $ filtron -rules rules.json - -It listens on ``127.0.0.1:4004`` and forwards filtered requests to -``127.0.0.1:8888`` by default. - -Use it along with ``nginx`` with the following example configuration. - -.. code:: nginx - - # https://example.org/searx - - location /searx { - proxy_pass http://127.0.0.1:4004/; - - proxy_set_header Host $host; - proxy_set_header Connection $http_connection; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Scheme $scheme; - proxy_set_header X-Script-Name /searx; - } - - location /searx/static { - /usr/local/searx/searx-src/searx/static; - } - - -Requests are coming from port 4004 going through filtron and then forwarded to -port 8888 where a SearXNG is being run. For a complete setup see: :ref:`nginx -searxng site`. diff --git a/docs/admin/index.rst b/docs/admin/index.rst index e6c0636b2..705167306 100644 --- a/docs/admin/index.rst +++ b/docs/admin/index.rst @@ -7,17 +7,15 @@ Administrator documentation :caption: Contents installation + installation-docker + installation-scripts installation-searxng installation-uwsgi installation-nginx installation-apache - installation-docker - installation-switch2ng update-searxng engines/index api architecture - filtron - morty plugins buildhosts diff --git a/docs/admin/installation-apache.rst b/docs/admin/installation-apache.rst index b60e20ad1..673a37ee4 100644 --- a/docs/admin/installation-apache.rst +++ b/docs/admin/installation-apache.rst @@ -1,13 +1,13 @@ .. _installation apache: -=================== -Install with apache -=================== +====== +Apache +====== .. _Apache: https://httpd.apache.org/ .. _Apache Debian: https://cwiki.apache.org/confluence/display/HTTPD/DistrosDefaultLayout#DistrosDefaultLayout-Debian,Ubuntu(Apachehttpd2.x): -.. _README.Debian: +.. _apache2.README.Debian: https://salsa.debian.org/apache-team/apache2/raw/master/debian/apache2.README.Debian .. _Apache Arch Linux: https://wiki.archlinux.org/index.php/Apache_HTTP_Server @@ -23,7 +23,9 @@ Install with apache https://httpd.apache.org/docs/current/en/configuring.html .. _ProxyPreserveHost: https://httpd.apache.org/docs/trunk/mod/mod_proxy.html#proxypreservehost .. _LoadModule: - https://httpd.apache.org/docs/2.4/mod/mod_so.html#loadmodule + https://httpd.apache.org/docs/mod/mod_so.html#loadmodule +.. _IncludeOptional: + https://httpd.apache.org/docs/mod/core.html#includeoptional .. _DocumentRoot: https://httpd.apache.org/docs/trunk/mod/core.html#documentroot .. _Location: @@ -32,11 +34,30 @@ Install with apache https://uwsgi-docs.readthedocs.io/en/latest/Apache.html .. _mod_proxy_uwsgi: https://uwsgi-docs.readthedocs.io/en/latest/Apache.html#mod-proxy-uwsgi +.. _mod_proxy_http: + https://httpd.apache.org/docs/current/mod/mod_proxy_http.html +.. _mod_proxy: + https://httpd.apache.org/docs/current/mod/mod_proxy.html + + +This section explains how to set up a SearXNG instance using the HTTP server Apache_. +If you did use the :ref:`installation scripts` and do not have any special preferences +you can install the :ref:`SearXNG site <apache searxng site>` using +:ref:`searxng.sh <searxng.sh overview>`: + +.. code:: bash + + $ sudo -H ./utils/searxng.sh install apache + +If you have special interests or problems with setting up Apache, the following +section might give you some guidance. + .. sidebar:: further read - `Apache Arch Linux`_ - - `Apache Debian`_ and `README.Debian`_ + - `Apache Debian`_ + - `apache2.README.Debian`_ - `Apache Fedora`_ - `Apache directives`_ @@ -45,23 +66,8 @@ Install with apache :local: :backlinks: entry ----- - -**Install** :ref:`apache searxng site` using :ref:`filtron.sh <filtron.sh overview>` - -.. code:: bash - - $ sudo -H ./utils/filtron.sh apache install - -**Install** :ref:`apache searxng site` using :ref:`morty.sh <morty.sh overview>` - -.. code:: bash - $ sudo -H ./utils/morty.sh apache install - ----- - -The apache HTTP server +The Apache HTTP server ====================== If Apache_ is not installed, install it now. If apache_ is new to you, the @@ -73,13 +79,13 @@ Directives`_ documentation gives first orientation. There is also a list of .. group-tab:: Ubuntu / debian - .. code:: sh + .. code:: bash sudo -H apt-get install apache2 .. group-tab:: Arch Linux - .. code:: sh + .. code:: bash sudo -H pacman -S apache sudo -H systemctl enable httpd @@ -87,21 +93,21 @@ Directives`_ documentation gives first orientation. There is also a list of .. group-tab:: Fedora / RHEL - .. code:: sh + .. code:: bash sudo -H dnf install httpd sudo -H systemctl enable httpd sudo -H systemctl start httpd -Now at http://localhost you should see any kind of *Welcome* or *Test* page. -How this default intro site is configured, depends on the linux distribution +Now at http://localhost you should see some kind of *Welcome* or *Test* page. +How this default site is configured, depends on the linux distribution (compare `Apache directives`_). .. tabs:: .. group-tab:: Ubuntu / debian - .. code:: sh + .. code:: bash less /etc/apache2/sites-enabled/000-default.conf @@ -115,7 +121,7 @@ How this default intro site is configured, depends on the linux distribution .. group-tab:: Arch Linux - .. code:: sh + .. code:: bash less /etc/httpd/conf/httpd.conf @@ -130,8 +136,8 @@ How this default intro site is configured, depends on the linux distribution Require all granted </Directory> - The *welcome* page of Arch Linux is a page showing directory located at - ``DocumentRoot``. This is *directory* page is generated by the Module + The *welcome* page of Arch Linux is a page showing the directory located + at ``DocumentRoot``. This *directory* page is generated by the Module `mod_autoindex <https://httpd.apache.org/docs/2.4/mod/mod_autoindex.html>`_: .. code:: apache @@ -142,7 +148,7 @@ How this default intro site is configured, depends on the linux distribution .. group-tab:: Fedora / RHEL - .. code:: sh + .. code:: bash less /etc/httpd/conf/httpd.conf @@ -163,323 +169,204 @@ How this default intro site is configured, depends on the linux distribution less /etc/httpd/conf.d/welcome.conf -.. _apache searxng site: -Apache Reverse Proxy -==================== +.. _Debian's Apache layout: -.. sidebar:: public to the internet? +Debian's Apache layout +---------------------- - If your SearXNG instance is public, stop here and first install :ref:`filtron - reverse proxy <filtron.sh>` and :ref:`result proxy morty <morty.sh>`, see - :ref:`installation scripts`. If already done, follow setup: *SearXNG via - filtron plus morty*. +Be aware, Debian's Apache layout is quite different from the standard Apache +configuration. For details look at the apache2.README.Debian_ +(``/usr/share/doc/apache2/README.Debian.gz``). Some commands you should know on +Debian: -To setup a Apache revers proxy you have to enable the *headers* and *proxy* -modules and create a `Location`_ configuration for the SearXNG site. In most -distributions you have to un-comment the lines in the main configuration file, -except in :ref:`The Debian Layout`. +* :man:`apache2ctl`: Apache HTTP server control interface +* :man:`a2enmod`, :man:`a2dismod`: switch on/off modules +* :man:`a2enconf`, :man:`a2disconf`: switch on/off configurations +* :man:`a2ensite`, :man:`a2dissite`: switch on/off sites + +.. _apache modules: + +Apache modules +-------------- + +To load additional modules, in most distributions you have to un-comment the +lines with the corresponding LoadModule_ directive, except in :ref:`Debian's +Apache layout`. .. tabs:: .. group-tab:: Ubuntu / debian - In the Apache setup, enable headers and proxy modules: + :ref:`Debian's Apache layout` uses :man:`a2enmod` and :man:`a2dismod` to + activate or disable modules: - .. code:: sh + .. code:: bash + sudo -H a2enmod ssl sudo -H a2enmod headers sudo -H a2enmod proxy sudo -H a2enmod proxy_http - - In :ref:`The Debian Layout` you create a ``searxng.conf`` with the - ``<Location /searx >`` directive and save this file in the *sites - available* folder at ``/etc/apache2/sites-available``. To enable the - ``searxng.conf`` use :man:`a2ensite`: - - .. code:: sh - - sudo -H a2ensite searxng.conf + sudo -H a2enmod proxy_uwsgi .. group-tab:: Arch Linux - In the ``/etc/httpd/conf/httpd.conf`` file, activate headers and proxy - modules (LoadModule_): + In the ``/etc/httpd/conf/httpd.conf`` file, activate LoadModule_ + directives: .. code:: apache - FIXME needs test - - LoadModule headers_module modules/mod_headers.so - LoadModule proxy_module modules/mod_proxy.so - LoadModule proxy_http_module modules/mod_proxy_http.so + LoadModule ssl_module modules/mod_ssl.so + LoadModule headers_module modules/mod_headers.so + LoadModule proxy_module modules/mod_proxy.so + LoadModule proxy_http_module modules/mod_proxy_http.so + LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so .. group-tab:: Fedora / RHEL - In the ``/etc/httpd/conf/httpd.conf`` file, activate headers and proxy - modules (LoadModule_): + In the ``/etc/httpd/conf/httpd.conf`` file, activate LoadModule_ + directives: .. code:: apache - FIXME needs test + LoadModule ssl_module modules/mod_ssl.so + LoadModule headers_module modules/mod_headers.so + LoadModule proxy_module modules/mod_proxy.so + LoadModule proxy_http_module modules/mod_proxy_http.so + LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so - LoadModule headers_module modules/mod_headers.so - LoadModule proxy_module modules/mod_proxy.so - LoadModule proxy_http_module modules/mod_proxy_http.so -With ProxyPreserveHost_ the incoming Host HTTP request header is passed to the -proxied host. +.. _apache sites: -.. _apache searxng via filtron plus morty: - -.. tabs:: - - .. group-tab:: SearXNG via filtron plus morty - - Use this setup, if your instance is public to the internet, compare - figure: :ref:`architecture <arch public>` and :ref:`installation scripts`. - - 1. Configure a reverse proxy for :ref:`filtron <filtron.sh>`, listening on - *localhost 4004* (:ref:`filtron route request`): - - .. code:: apache - - <Location /searx > - - # SetEnvIf Request_URI "/searx" dontlog - # CustomLog /dev/null combined env=dontlog - - Require all granted - - Order deny,allow - Deny from all - #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 - Allow from all - - ProxyPreserveHost On - ProxyPass http://127.0.0.1:4004 - RequestHeader set X-Script-Name /searx - - </Location> - - 2. Configure reverse proxy for :ref:`morty <searxng morty>`, listening on - *localhost 3000* - - .. code:: apache - - ProxyPreserveHost On - - <Location /morty > - - # SetEnvIf Request_URI "/morty" dontlog - # CustomLog /dev/null combined env=dontlog - - Require all granted - - Order deny,allow - Deny from all - #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 - Allow from all - - ProxyPass http://127.0.0.1:3000 - RequestHeader set X-Script-Name /morty - - </Location> - - For a fully result proxification add :ref:`morty's <searxng morty>` **public - URL** to your :origin:`searx/settings.yml`: - - .. code:: yaml - - result_proxy: - # replace example.org with your server's public name - url : https://example.org/morty - key : !!binary "insert_your_morty_proxy_key_here" - - server: - image_proxy : True - -uWSGI support -============= - -Be warned, with this setup, your instance isn't :ref:`protected <searxng -filtron>`, nevertheless it is good enough for intranet usage. In modern Linux -distributions, the `mod_proxy_uwsgi`_ is compiled into the *normal* apache -package and you need to install only the :ref:`uWSGI <searxng uwsgi>` package: +Apache sites +------------ .. tabs:: .. group-tab:: Ubuntu / debian - .. code:: sh + In :ref:`Debian's Apache layout` you create a ``searxng.conf`` with the + ``<Location /searxng >`` directive and save this file in the *sites + available* folder at ``/etc/apache2/sites-available``. To enable the + ``searxng.conf`` use :man:`a2ensite`: - sudo -H apt-get install uwsgi + .. code:: bash - # Ubuntu =< 18.04 - sudo -H apt-get install libapache2-mod-proxy-uwsgi + sudo -H a2ensite searxng.conf .. group-tab:: Arch Linux - .. code:: sh - - sudo -H pacman -S uwsgi - - .. group-tab:: Fedora / RHEL - - .. code:: sh - - sudo -H dnf install uwsgi - -The next example shows a configuration using the `uWSGI Apache support`_ via -unix sockets and `mod_proxy_uwsgi`_. - -For socket communication, you have to activate ``socket = -/run/uwsgi/app/searx/socket`` and comment out the ``http = 127.0.0.1:8888`` -configuration in your :ref:`uwsgi ini file <uwsgi configuration>`. If not -already exists, create a folder for the unix sockets, which can be used by the -SearXNG account (see :ref:`create searxng user`): - -.. code:: bash - - sudo -H mkdir -p /run/uwsgi/app/searx/ - sudo -H chown -R searx:searx /run/uwsgi/app/searx/ - -If the server is public; to limit access to your intranet replace ``Allow from -all`` directive and replace ``192.168.0.0/16`` with your subnet IP/class. - -.. tabs:: - - .. group-tab:: Ubuntu / debian + In the ``/etc/httpd/conf/httpd.conf`` file add a IncludeOptional_ + directive: .. code:: apache - LoadModule headers_module /usr/lib/apache2/mod_headers.so - LoadModule proxy_module /usr/lib/apache2/modules/mod_proxy.so - LoadModule proxy_uwsgi_module /usr/lib/apache2/modules/mod_proxy_uwsgi.so - - # SetEnvIf Request_URI /searx dontlog - # CustomLog /dev/null combined env=dontlog + IncludeOptional sites-enabled/*.conf - <Location /searx> + Create two folders, one for the *available sites* and one for the *enabled sites*: - Require all granted - Order deny,allow - Deny from all - # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 - Allow from all + .. code:: bash - ProxyPreserveHost On - ProxyPass unix:/run/uwsgi/app/searx/socket|uwsgi://uwsgi-uds-searx/ + mkdir -p /etc/httpd/sites-available + mkdir -p /etc/httpd/sites-enabled - </Location> + Create configuration at ``/etc/httpd/sites-available`` and place a + symlink to ``sites-enabled``: - .. group-tab:: Arch Linux - - .. code:: apache + .. code:: bash - FIXME needs test + sudo -H ln -s /etc/httpd/sites-available/searxng.conf \ + /etc/httpd/sites-enabled/searxng.conf - LoadModule proxy_module modules/mod_proxy.so - LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so - - # SetEnvIf Request_URI /searx dontlog - # CustomLog /dev/null combined env=dontlog + .. group-tab:: Fedora / RHEL - <Location /searx> + In the ``/etc/httpd/conf/httpd.conf`` file add a IncludeOptional_ + directive: - Require all granted - Order deny,allow - Deny from all - # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 - Allow from all + .. code:: apache - ProxyPreserveHost On - ProxyPass unix:/run/uwsgi/app/searx/socket|uwsgi://uwsgi-uds-searx/ + IncludeOptional sites-enabled/*.conf - </Location> + Create two folders, one for the *available sites* and one for the *enabled sites*: - .. group-tab:: Fedora / RHEL + .. code:: bash - .. code:: apache + mkdir -p /etc/httpd/sites-available + mkdir -p /etc/httpd/sites-enabled - FIXME needs test + Create configuration at ``/etc/httpd/sites-available`` and place a + symlink to ``sites-enabled``: - LoadModule proxy_module modules/mod_proxy.so - LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so - <IfModule proxy_uwsgi_module> + .. code:: bash - # SetEnvIf Request_URI /searx dontlog - # CustomLog /dev/null combined env=dontlog + sudo -H ln -s /etc/httpd/sites-available/searxng.conf \ + /etc/httpd/sites-enabled/searxng.conf - <Location /searx> - Require all granted - Order deny,allow - Deny from all - # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 - Allow from all +.. _apache searxng site: - ProxyPreserveHost On - ProxyPass unix:/run/uwsgi/app/searx/socket|uwsgi://uwsgi-uds-searx/ +Apache's SearXNG site +===================== - </Location> +.. _mod_uwsgi: https://uwsgi-docs.readthedocs.io/en/latest/Apache.html#mod-uwsgi - </IfModule> +.. sidebar:: uWSGI - .. group-tab:: old mod_wsgi + Use mod_proxy_uwsgi_ / don't use the old mod_uwsgi_ anymore. - We show this only for historical reasons, DON'T USE `mod_uwsgi - <https://uwsgi-docs.readthedocs.io/en/latest/Apache.html#mod-uwsgi>`_. - ANYMORE! +To proxy the incoming requests to the SearXNG instance Apache needs the +mod_proxy_ module (:ref:`apache modules`). - .. code:: apache +.. sidebar:: HTTP headers - <IfModule mod_uwsgi.c> + With ProxyPreserveHost_ the incoming ``Host`` header is passed to the proxied + host. - # SetEnvIf Request_URI "/searx" dontlog - # CustomLog /dev/null combined env=dontlog +Depending on what your SearXNG installation is listening on, you need a http +mod_proxy_http_) or socket (mod_proxy_uwsgi_) communication to upstream. - <Location /searx > +The :ref:`installation scripts` installs the :ref:`reference setup +<use_default_settings.yml>` and a :ref:`uwsgi setup` that listens on a socket by default. +You can install and activate your own ``searxng.conf`` like shown in +:ref:`apache sites`. - Require all granted +.. tabs:: - Options FollowSymLinks Indexes - SetHandler uwsgi-handler - uWSGISocket /run/uwsgi/app/searx/socket + .. group-tab:: socket - Order deny,allow - Deny from all - # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 - Allow from all + .. kernel-include:: $DOCS_BUILD/includes/searxng.rst + :start-after: START apache socket + :end-before: END apache socket - </Location> + .. group-tab:: http - </IfModule> + .. kernel-include:: $DOCS_BUILD/includes/searxng.rst + :start-after: START apache http + :end-before: END apache http .. _restart apache: -Restart service -=============== +Restart service: .. tabs:: .. group-tab:: Ubuntu / debian - .. code:: sh + .. code:: bash sudo -H systemctl restart apache2 - sudo -H service uwsgi restart searx + sudo -H service uwsgi restart searxng .. group-tab:: Arch Linux - .. code:: sh + .. code:: bash sudo -H systemctl restart httpd - sudo -H systemctl restart uwsgi@searx + sudo -H systemctl restart uwsgi@searxng .. group-tab:: Fedora / RHEL - .. code:: sh + .. code:: bash sudo -H systemctl restart httpd sudo -H touch /etc/uwsgi.d/searxng.ini @@ -489,27 +376,13 @@ disable logs ============ For better privacy you can disable Apache logs. In the examples above activate -one of the lines and `restart apache`_:: - +one of the lines and `restart apache`_: - # SetEnvIf Request_URI "/searx" dontlog - # CustomLog /dev/null combined env=dontlog +.. code:: apache -The ``CustomLog`` directive disable logs for the whole (virtual) server, use it -when the URL of the service does not have a path component (``/searx``) / is -located at root (``/``). + SetEnvIf Request_URI "/searxng" dontlog + # CustomLog /dev/null combined env=dontlog -.. _The Debian Layout: - -The Debian Layout -================= - -Be aware that the Debian layout is quite different from the standard Apache -configuration. For details look at the README.Debian_ -(``/usr/share/doc/apache2/README.Debian.gz``). Some commands you should know on -Debian: - -* :man:`apache2ctl`: Apache HTTP server control interface -* :man:`a2enmod`, :man:`a2dismod`: switch on/off modules -* :man:`a2enconf`, :man:`a2disconf`: switch on/off configurations -* :man:`a2ensite`, :man:`a2dissite`: switch on/off sites +The ``CustomLog`` directive disables logs for the entire (virtual) server, use it +when the URL of the service does not have a path component (``/searxng``), so when +SearXNG is located at root (``/``). diff --git a/docs/admin/installation-docker.rst b/docs/admin/installation-docker.rst index cd91af382..1457d6a6b 100644 --- a/docs/admin/installation-docker.rst +++ b/docs/admin/installation-docker.rst @@ -1,37 +1,60 @@ - .. _installation docker: -=================== -Docker installation -=================== +================ +Docker Container +================ .. _ENTRYPOINT: https://docs.docker.com/engine/reference/builder/#entrypoint +.. _searxng/searxng @dockerhub: https://hub.docker.com/r/searxng/searxng .. _searxng-docker: https://github.com/searxng/searxng-docker -.. _[filtron]: https://hub.docker.com/r/dalf/filtron -.. _[morty]: https://hub.docker.com/r/dalf/morty .. _[caddy]: https://hub.docker.com/_/caddy +.. _Redis: https://redis.io/ + +---- .. sidebar:: info + - `searxng/searxng @dockerhub`_ - :origin:`Dockerfile` - - `searxng/searxng @dockerhub <https://hub.docker.com/r/searxng/searxng>`_ - `Docker overview <https://docs.docker.com/get-started/overview>`_ - - `Docker Cheat Sheet <https://www.docker.com/sites/default/files/d8/2019-09/docker-cheat-sheet.pdf>`_ - - `Alpine Linux <https://alpinelinux.org>`_ `(wiki) <https://en.wikipedia.org/wiki/Alpine_Linux>`__ `apt packages <https://pkgs.alpinelinux.org/packages>`_ + - `Docker Cheat Sheet <https://docs.docker.com/get-started/docker_cheatsheet.pdf>`_ + - `Alpine Linux <https://alpinelinux.org>`_ + `(wiki) <https://en.wikipedia.org/wiki/Alpine_Linux>`__ + `apt packages <https://pkgs.alpinelinux.org/packages>`_ - Alpine's ``/bin/sh`` is :man:`dash` -.. tip:: +**If you intend to create a public instance using Docker, use our well maintained +docker container** + +- `searxng/searxng @dockerhub`_. + +.. sidebar:: hint + + The rest of this article is of interest only to those who want to create and + maintain their own Docker images. + +The sources are hosted at searxng-docker_ and the container includes: + +- a HTTPS reverse proxy `[caddy]`_ and +- a Redis_ DB + +The `default SearXNG setup <https://github.com/searxng/searxng-docker/blob/master/searxng/settings.yml>`_ +of this container: + +- enables :ref:`limiter <limiter>` to protect against bots +- enables :ref:`image proxy <image_proxy>` for better privacy +- enables :ref:`cache busting <static_use_hash>` to save bandwith + +---- - If you intend to create a public instance using Docker, use our well - maintained searxng-docker_ image which includes - - :ref:`protection <searxng filtron>` `[filtron]`_, - - a :ref:`result proxy <searxng morty>` `[morty]`_ and - - a HTTPS reverse proxy `[caddy]`_. +Get Docker +========== -Make sure you have `installed Docker <https://docs.docker.com/get-docker/>`_ and -on Linux, don't forget to add your user to the docker group (log out and log -back in so that your group membership is re-evaluated): +If you plan to build and maintain a docker image by yourself, make sure you have +`Docker installed <https://docs.docker.com/get-docker/>`_. On Linux don't +forget to add your user to the docker group (log out and log back in so that +your group membership is re-evaluated): .. code:: sh diff --git a/docs/admin/installation-nginx.rst b/docs/admin/installation-nginx.rst index fdbcdf032..8e5299588 100644 --- a/docs/admin/installation-nginx.rst +++ b/docs/admin/installation-nginx.rst @@ -1,8 +1,8 @@ .. _installation nginx: -================== -Install with nginx -================== +===== +NGINX +===== .. _nginx: https://docs.nginx.com/nginx/admin-guide/ @@ -19,6 +19,19 @@ Install with nginx .. _SCRIPT_NAME: https://werkzeug.palletsprojects.com/en/1.0.x/wsgi/#werkzeug.wsgi.get_script_name +This section explains how to set up a SearXNG instance using the HTTP server nginx_. +If you have used the :ref:`installation scripts` and do not have any special preferences +you can install the :ref:`SearXNG site <nginx searxng site>` using +:ref:`searxng.sh <searxng.sh overview>`: + +.. code:: bash + + $ sudo -H ./utils/searxng.sh install nginx + +If you have special interests or problems with setting up nginx, the following +section might give you some guidance. + + .. sidebar:: further reading - nginx_ @@ -27,39 +40,23 @@ Install with nginx - `Getting Started wiki`_ - `uWSGI support from nginx`_ + .. contents:: Contents :depth: 2 :local: :backlinks: entry ----- - -**Install** :ref:`nginx searxng site` using :ref:`filtron.sh <filtron.sh overview>` - -.. code:: bash - - $ sudo -H ./utils/filtron.sh nginx install - -**Install** :ref:`nginx searxng site` using :ref:`morty.sh <morty.sh overview>` - -.. code:: bash - - $ sudo -H ./utils/morty.sh nginx install - ----- - The nginx HTTP server ===================== -If nginx_ is not installed (uwsgi will not work with the package nginx-light), -install it now. +If nginx_ is not installed, install it now. .. tabs:: .. group-tab:: Ubuntu / debian - .. code:: sh + .. code:: bash sudo -H apt-get install nginx @@ -81,18 +78,18 @@ install it now. Now at http://localhost you should see a *Welcome to nginx!* page, on Fedora you see a *Fedora Webserver - Test Page*. The test page comes from the default -`nginx server configuration`_. How this default intro site is configured, +`nginx server configuration`_. How this default site is configured, depends on the linux distribution: .. tabs:: .. group-tab:: Ubuntu / debian - .. code:: sh + .. code:: bash less /etc/nginx/nginx.conf - there is a line including site configurations from: + There is one line that includes site configurations from: .. code:: nginx @@ -104,7 +101,7 @@ depends on the linux distribution: less /etc/nginx/nginx.conf - in there is a configuration section named ``server``: + There is a configuration section named ``server``: .. code-block:: nginx @@ -120,249 +117,121 @@ depends on the linux distribution: less /etc/nginx/nginx.conf - there is a line including site configurations from: + There is one line that includes site configurations from: .. code:: nginx include /etc/nginx/conf.d/*.conf; + .. _nginx searxng site: -A nginx SearXNG site +NGINX's SearXNG site ==================== -.. sidebar:: public to the internet? - - If your SearXNG instance is public, stop here and first install :ref:`filtron - reverse proxy <filtron.sh>` and :ref:`result proxy morty <morty.sh>`, see - :ref:`installation scripts`. If already done, follow setup: *SearXNG via - filtron plus morty*. +Now you have to create a configuration file (``searxng.conf``) for the SearXNG +site. If nginx_ is new to you, the `nginx beginners guide`_ is a good starting +point and the `Getting Started wiki`_ is always a good resource *to keep in the +pocket*. -Now you have to create a configuration for the SearXNG site. If nginx_ is new to -you, the `nginx beginners guide`_ is a good starting point and the `Getting -Started wiki`_ is always a good resource *to keep in the pocket*. +Depending on what your SearXNG installation is listening on, you need a http or socket +communication to upstream. .. tabs:: - .. group-tab:: Ubuntu / debian - - Create configuration at ``/etc/nginx/sites-available/searxng`` and place a - symlink to sites-enabled: - - .. code:: sh - - sudo -H ln -s /etc/nginx/sites-available/searxng /etc/nginx/sites-enabled/searxng - - .. group-tab:: Arch Linux + .. group-tab:: socket - In the ``/etc/nginx/nginx.conf`` file, replace the configuration section - named ``server``. + .. kernel-include:: $DOCS_BUILD/includes/searxng.rst + :start-after: START nginx socket + :end-before: END nginx socket - .. group-tab:: Fedora / RHEL + .. group-tab:: http - Create configuration at ``/etc/nginx/conf.d/searxng`` and place a - symlink to sites-enabled: + .. kernel-include:: $DOCS_BUILD/includes/searxng.rst + :start-after: START nginx http + :end-before: END nginx http -.. _nginx searxng via filtron plus morty: +The :ref:`installation scripts` installs the :ref:`reference setup +<use_default_settings.yml>` and a :ref:`uwsgi setup` that listens on a socket by default. .. tabs:: - .. group-tab:: SearXNG via filtron plus morty - - Use this setup, if your instance is public to the internet, compare - figure: :ref:`architecture <arch public>` and :ref:`installation scripts`. - - 1. Configure a reverse proxy for :ref:`filtron <filtron.sh>`, listening on - *localhost 4004* (:ref:`filtron route request`): - - .. code:: nginx - - # https://example.org/searx - - location /searx { - proxy_pass http://127.0.0.1:4004/; - - proxy_set_header Host $host; - proxy_set_header Connection $http_connection; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Scheme $scheme; - proxy_set_header X-Script-Name /searx; - } - - location /searx/static/ { - alias /usr/local/searx/searx-src/searx/static/; - } - - - 2. Configure reverse proxy for :ref:`morty <searxng morty>`, listening on - *localhost 3000*: - - .. code:: nginx - - # https://example.org/morty - - location /morty { - proxy_pass http://127.0.0.1:3000/; - - proxy_set_header Host $host; - proxy_set_header Connection $http_connection; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Scheme $scheme; - } - - For a fully result proxification add :ref:`morty's <searxng morty>` **public - URL** to your :origin:`searx/settings.yml`: - - .. code:: yaml - - result_proxy: - # replace example.org with your server's public name - url : https://example.org/morty - key : !!binary "insert_your_morty_proxy_key_here" - - server: - image_proxy : True - - - .. group-tab:: proxy or uWSGI - - Be warned, with this setup, your instance isn't :ref:`protected <searxng - filtron>`. Nevertheless it is good enough for intranet usage and it is a - excellent example of; *how different services can be set up*. The next - example shows a reverse proxy configuration wrapping the :ref:`searx-uWSGI - application <uwsgi configuration>`, listening on ``http = - 127.0.0.1:8888``. - - .. code:: nginx + .. group-tab:: Ubuntu / debian - # https://hostname.local/ + Create configuration at ``/etc/nginx/sites-available/`` and place a + symlink to ``sites-enabled``: - location / { - proxy_pass http://127.0.0.1:8888; + .. code:: bash - proxy_set_header Host $host; - proxy_set_header Connection $http_connection; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Scheme $scheme; - proxy_buffering off; - } + sudo -H ln -s /etc/nginx/sites-available/searxng.conf \ + /etc/nginx/sites-enabled/searxng.conf - Alternatively you can use the `uWSGI support from nginx`_ via unix - sockets. For socket communication, you have to activate ``socket = - /run/uwsgi/app/searx/socket`` and comment out the ``http = - 127.0.0.1:8888`` configuration in your :ref:`uwsgi ini file <uwsgi - configuration>`. + .. group-tab:: Arch Linux - The example shows a nginx virtual ``server`` configuration, listening on - port 80 (IPv4 and IPv6 http://[::]:80). The uWSGI app is configured at - location ``/`` by importing the `uwsgi_params`_ and passing requests to - the uWSGI socket (``uwsgi_pass``). The ``server``\'s root points to the - :ref:`searx-src clone <searx-src>` and wraps directly the - :origin:`searx/static/` content at ``location /static``. + In the ``/etc/nginx/nginx.conf`` file, in the ``server`` section add a + `include <https://nginx.org/en/docs/ngx_core_module.html#include>`_ + directive: .. code:: nginx server { - # replace hostname.local with your server's name - server_name hostname.local; - - listen 80; - listen [::]:80; - - location / { - include uwsgi_params; - uwsgi_pass unix:/run/uwsgi/app/searx/socket; - } - - root /usr/local/searx/searx-src/searx; - location /static { } + # ... + include /etc/nginx/default.d/*.conf; + # ... } - If not already exists, create a folder for the unix sockets, which can be - used by the SearXNG account: + Create two folders, one for the *available sites* and one for the *enabled sites*: .. code:: bash - mkdir -p /run/uwsgi/app/searx/ - sudo -H chown -R searx:searx /run/uwsgi/app/searx/ - - .. group-tab:: \.\. at subdir URL - - Be warned, with these setups, your instance isn't :ref:`protected <searxng - filtron>`. The examples are just here to demonstrate how to export the - SearXNG application from a subdirectory URL ``https://example.org/searx/``. - - .. code:: nginx - - # https://hostname.local/searx - - location /searx { - proxy_pass http://127.0.0.1:8888; + mkdir -p /etc/nginx/default.d + mkdir -p /etc/nginx/default.apps-available - proxy_set_header Host $host; - proxy_set_header Connection $http_connection; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Scheme $scheme; - proxy_set_header X-Script-Name /searx; - proxy_buffering off; - } - - location /searx/static/ { - alias /usr/local/searx/searx-src/searx/static/; - } + Create configuration at ``/etc/nginx/default.apps-available`` and place a + symlink to ``default.d``: - The ``X-Script-Name /searx`` is needed by the SearXNG implementation to - calculate relative URLs correct. The next example shows a uWSGI - configuration. Since there are no HTTP headers in a (u)WSGI protocol, the - value is shipped via the SCRIPT_NAME_ in the WSGI environment. + .. code:: bash - .. code:: nginx + sudo -H ln -s /etc/nginx/default.apps-available/searxng.conf \ + /etc/nginx/default.d/searxng.conf - # https://hostname.local/searx + .. group-tab:: Fedora / RHEL - location /searx { - uwsgi_param SCRIPT_NAME /searx; - include uwsgi_params; - uwsgi_pass unix:/run/uwsgi/app/searx/socket; - } + Create a folder for the *available sites*: - location /searx/static/ { - alias /usr/local/searx/searx-src/searx/; - } + .. code:: bash - For SearXNG to work correctly the ``base_url`` must be set in the - :origin:`searx/settings.yml`. + mkdir -p /etc/nginx/default.apps-available - .. code:: yaml + Create configuration at ``/etc/nginx/default.apps-available`` and place a + symlink to ``conf.d``: - server: - # replace example.org with your server's public name - base_url : https://example.org/searx/ + .. code:: bash + sudo -H ln -s /etc/nginx/default.apps-available/searxng.conf \ + /etc/nginx/conf.d/searxng.conf -Restart service: +Restart services: .. tabs:: .. group-tab:: Ubuntu / debian - .. code:: sh + .. code:: bash sudo -H systemctl restart nginx - sudo -H service uwsgi restart searx + sudo -H service uwsgi restart searxng .. group-tab:: Arch Linux - .. code:: sh + .. code:: bash sudo -H systemctl restart nginx - sudo -H systemctl restart uwsgi@searx + sudo -H systemctl restart uwsgi@searxng - .. group-tab:: Fedora + .. group-tab:: Fedora / RHEL - .. code:: sh + .. code:: bash sudo -H systemctl restart nginx sudo -H touch /etc/uwsgi.d/searxng.ini diff --git a/docs/admin/installation-scripts.rst b/docs/admin/installation-scripts.rst new file mode 100644 index 000000000..e256a2461 --- /dev/null +++ b/docs/admin/installation-scripts.rst @@ -0,0 +1,62 @@ +.. _installation scripts: + +=================== +Installation Script +=================== + +.. sidebar:: Update the OS first! + + To avoid unwanted side effects, update your OS before installing SearXNG. + +The following will install a setup as shown in :ref:`the reference architecture +<arch public>`. First you need to get a clone of the repository. The clone is only needed for +the installation procedure and some maintenance tasks. + +.. sidebar:: further read + + - :ref:`toolboxing` + +Jump to a folder that is readable by *others* and start to clone SearXNG, +alternatively you can create your own fork and clone from there. + +.. code:: bash + + $ cd ~/Downloads + $ git clone https://github.com/searxng/searxng.git searxng + $ cd searxng + +.. sidebar:: further read + + - :ref:`inspect searxng` + +To install a SearXNG :ref:`reference setup <use_default_settings.yml>` +including a :ref:`uWSGI setup <architecture uWSGI>` as described in the +:ref:`installation basic` and in the :ref:`searxng uwsgi` section type: + +.. code:: bash + + $ sudo -H ./utils/searxng.sh install all + +.. attention:: + + For the installation procedure, use a *sudoer* login to run the scripts. If + you install from ``root``, take into account that the scripts are creating a + ``searxng`` user. In the installation procedure this new created user does + need read access to the cloned SearXNG repository, which is not the case if you clone + it into a folder below ``/root``! + +.. sidebar:: further read + + - :ref:`update searxng` + +.. _caddy: https://hub.docker.com/_/caddy + +When all services are installed and running fine, you can add SearXNG to your +HTTP server. We do not have any preferences for the HTTP server, you can use +whatever you prefer. + +We use caddy in our :ref:`docker image <installation docker>` and we have +implemented installation procedures for: + +- :ref:`installation nginx` +- :ref:`installation apache` diff --git a/docs/admin/installation-searxng.rst b/docs/admin/installation-searxng.rst index b14139310..9152784fc 100644 --- a/docs/admin/installation-searxng.rst +++ b/docs/admin/installation-searxng.rst @@ -9,15 +9,16 @@ Step by step installation :local: :backlinks: entry -Step by step installation with virtualenv. For Ubuntu, be sure to have enable -universe repository. + +In this section we show the setup of a SearXNG instance that will be installed +by the :ref:`installation scripts`. .. _install packages: Install packages ================ -.. kernel-include:: $DOCS_BUILD/includes/searx.rst +.. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START distro-packages :end-before: END distro-packages @@ -30,32 +31,32 @@ Install packages Create user =========== -.. kernel-include:: $DOCS_BUILD/includes/searx.rst +.. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START create user :end-before: END create user -.. _searx-src: +.. _searxng-src: Install SearXNG & dependencies ============================== -Start a interactive shell from new created user and clone searx: +Start a interactive shell from new created user and clone SearXNG: -.. kernel-include:: $DOCS_BUILD/includes/searx.rst +.. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START clone searxng :end-before: END clone searxng In the same shell create *virtualenv*: -.. kernel-include:: $DOCS_BUILD/includes/searx.rst +.. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START create virtualenv :end-before: END create virtualenv -To install searx's dependencies, exit the SearXNG *bash* session you opened above -and restart a new. Before install, first check if your *virtualenv* was sourced +To install SearXNG's dependencies, exit the SearXNG *bash* session you opened above +and start a new one. Before installing, check if your *virtualenv* was sourced from the login (*~/.profile*): -.. kernel-include:: $DOCS_BUILD/includes/searx.rst +.. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START manage.sh update_packages :end-before: END manage.sh update_packages @@ -77,30 +78,41 @@ Configuration - :ref:`settings use_default_settings` - :origin:`/etc/searxng/settings.yml <utils/templates/etc/searxng/settings.yml>` -To create a initial ``/etc/searxng/settings.yml`` you can start with a copy of -the file :origin:`utils/templates/etc/searxng/settings.yml`. This setup +To create a initial ``/etc/searxng/settings.yml`` we recommend to start with a +copy of the file :origin:`utils/templates/etc/searxng/settings.yml`. This setup :ref:`use default settings <settings use_default_settings>` from -:origin:`searx/settings.yml`. +:origin:`searx/settings.yml` and is shown in the tab *"Use default settings"* +below. This setup: -For a *minimal setup*, configure like shown below – replace ``searx@$(uname --n)`` with a name of your choice, set ``ultrasecretkey`` -- *and/or* edit -``/etc/searxng/settings.yml`` to your needs. +- enables :ref:`limiter <limiter>` to protect against bots +- enables :ref:`image proxy <image_proxy>` for better privacy +- enables :ref:`cache busting <static_use_hash>` to save bandwith -.. kernel-include:: $DOCS_BUILD/includes/searx.rst - :start-after: START searxng config - :end-before: END searxng config +Modify the ``/etc/searxng/settings.yml`` to your needs: .. tabs:: .. group-tab:: Use default settings - .. literalinclude:: ../../utils/templates/etc/searxng/settings.yml - :language: yaml + .. literalinclude:: ../../utils/templates/etc/searxng/settings.yml + :language: yaml + :end-before: # hostname_replace: + + To see the entire file jump to :origin:`utils/templates/etc/searxng/settings.yml` .. group-tab:: searx/settings.yml - .. literalinclude:: ../../searx/settings.yml - :language: yaml + .. literalinclude:: ../../searx/settings.yml + :language: yaml + :end-before: # hostname_replace: + + To see the entire file jump to :origin:`searx/settings.yml` + +For a *minimal setup* you need to set ``server:secret_key``. + +.. kernel-include:: $DOCS_BUILD/includes/searxng.rst + :start-after: START searxng config + :end-before: END searxng config Check @@ -110,11 +122,11 @@ To check your SearXNG setup, optional enable debugging and start the *webapp*. SearXNG looks at the exported environment ``$SEARXNG_SETTINGS_PATH`` for a configuration file. -.. kernel-include:: $DOCS_BUILD/includes/searx.rst +.. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START check searxng installation :end-before: END check searxng installation If everything works fine, hit ``[CTRL-C]`` to stop the *webapp* and disable the -debug option in ``settings.yml``. You can now exit SearXNG user bash (enter exit +debug option in ``settings.yml``. You can now exit SearXNG user bash session (enter exit command twice). At this point SearXNG is not demonized; uwsgi allows this. diff --git a/docs/admin/installation-switch2ng.rst b/docs/admin/installation-switch2ng.rst deleted file mode 100644 index 8863cd71e..000000000 --- a/docs/admin/installation-switch2ng.rst +++ /dev/null @@ -1,75 +0,0 @@ -.. _installation switch2ng: - -============================ -Switch from searx to SearXNG -============================ - -.. sidebar:: info - - - :pull:`456` - - :pull:`A comment about rolling release <446#issuecomment-954730358>` - -.. contents:: Contents - :depth: 2 - :local: - :backlinks: entry - -If you have a searx installation on your sever and want to switch to SearXNG, -you need to uninstall searx first. If you have an old searx docker installation -replace your docker image / see :ref:`installation docker`. - -If your searx instance was installed *"Step by step"* or by the *"Installation -scripts"*, you need to undo the installation procedure completely. If you have -morty & filtron installed, it is recommended to uninstall these services also. -In case of scripts, to uninstall use the scripts from the origin you installed -searx from. - -If you have removed the old searx installation, clone from SearXNG and and start -with your installation procedure (e.g. :ref:`installation scripts`): - -.. code:: bash - - $ cd ~/Downloads - $ git clone https://github.com/searxng/searxng.git searxng - $ cd searxng - $ ... - -``.config.sh`` -============== - -Please take into account; SearXNG has normalized ``.config.sh`` with -``settings.yml`` and some of the environment settings has been removed from or -renamed in the ``.config.sh``: - -- :patch:`[mod] normalize .config.sh with settings.yml <f61c918d>` -- :patch:`[fix] ./utils/filtron.sh - FILTRON_TARGET from YAML settings <7196a9b5>` -- :patch:`SearXNG: SEARXNG_SETTINGS_PATH <253b8503>` - - -Check after Installation -======================== - -Once you have done your installation, you can run a SearXNG *check* procedure, -to see if there are some left overs. In this example there exists a *old* -``/etc/searx/settings.yml``:: - - $ sudo -H ./utils/searx.sh install check - - ============================ - SearXNG (check installation) - ============================ - ERROR: settings.yml in /etc/searx/ is deprecated, move file to folder /etc/searxng/ - INFO: SearXNG instance already installed at: /usr/local/searx/searx-src - ... - INFO: Service account searx exists. - INFO: ~searx: python environment is available. - INFO: ~searx: SearXNG software is installed. - INFO: uWSGI app searxng.ini is enabled. - INFO searx : merge the default settings ( /usr/local/searx/searx-src/searx/settings.yml ) and the user setttings ( /etc/searxng/settings.yml ) - INFO searx : max_request_timeout=None - - -To *check* the filtron & morty installations, use similar commands:: - - $ sudo -H /utils/filtron.sh install check - $ sudo -H /utils/morty.sh install check diff --git a/docs/admin/installation-uwsgi.rst b/docs/admin/installation-uwsgi.rst index 1021283f9..a6ea85608 100644 --- a/docs/admin/installation-uwsgi.rst +++ b/docs/admin/installation-uwsgi.rst @@ -1,7 +1,7 @@ .. _searxng uwsgi: ===== -uwsgi +uWSGI ===== .. sidebar:: further reading @@ -29,51 +29,77 @@ uwsgi Origin uWSGI ============ -How uWSGI is implemented by distributors is different. uWSGI itself -recommend two methods +.. _Tyrant mode: + https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html#tyrant-mode-secure-multi-user-hosting -`systemd.unit`_ template files as described here `One service per app in systemd`_. +How uWSGI is implemented by distributors varies. The uWSGI project itself +recommends two methods: - There is one `systemd unit template`_ and one `uwsgi ini file`_ per uWSGI-app - placed at dedicated locations. Take archlinux and a searxng.ini as example:: +1. `systemd.unit`_ template file as described here `One service per app in systemd`_: - unit template --> /usr/lib/systemd/system/uwsgi@.service - uwsgi ini files --> /etc/uwsgi/searxng.ini + There is one `systemd unit template`_ on the system installed and one `uwsgi + ini file`_ per uWSGI-app placed at dedicated locations. Take archlinux and a + ``searxng.ini`` as example:: - The SearXNG app can be maintained as know from common systemd units:: + systemd template unit: /usr/lib/systemd/system/uwsgi@.service + contains: [Service] + ExecStart=/usr/bin/uwsgi --ini /etc/uwsgi/%I.ini - systemctl enable uwsgi@searx - systemctl start uwsgi@searx - systemctl restart uwsgi@searx - systemctl stop uwsgi@searx + SearXNG application: /etc/uwsgi/searxng.ini + links to: /etc/uwsgi/apps-available/searxng.ini -The `uWSGI Emperor`_ mode which fits for maintaining a large range of uwsgi apps. + The SearXNG app (template ``/etc/uwsgi/%I.ini``) can be maintained as known + from common systemd units: + + .. code:: sh + + $ systemctl enable uwsgi@searxng + $ systemctl start uwsgi@searxng + $ systemctl restart uwsgi@searxng + $ systemctl stop uwsgi@searxng + +2. The `uWSGI Emperor`_ which fits for maintaining a large range of uwsgi + apps and there is a `Tyrant mode`_ to secure multi-user hosting. The Emperor mode is a special uWSGI instance that will monitor specific - events. The Emperor mode (service) is started by a (common, not template) - systemd unit. The Emperor service will scan specific directories for `uwsgi - ini file`_\s (also know as *vassals*). If a *vassal* is added, removed or the - timestamp is modified, a corresponding action takes place: a new uWSGI - instance is started, reload or stopped. Take Fedora and a searxng.ini as - example:: + events. The Emperor mode (the service) is started by a (common, not template) + systemd unit. + + The Emperor service will scan specific directories for `uwsgi ini file`_\s + (also know as *vassals*). If a *vassal* is added, removed or the timestamp is + modified, a corresponding action takes place: a new uWSGI instance is started, + reload or stopped. Take Fedora and a ``searxng.ini`` as example:: + + to install & start SearXNG instance create --> /etc/uwsgi.d/searxng.ini + to reload the instance edit timestamp --> touch /etc/uwsgi.d/searxng.ini + to stop instance remove ini --> rm /etc/uwsgi.d/searxng.ini - to start a new SearXNG instance create --> /etc/uwsgi.d/searxng.ini - to reload the instance edit timestamp --> touch /etc/uwsgi.d/searxng.ini - to stop instance remove ini --> rm /etc/uwsgi.d/searxng.ini Distributors ============ The `uWSGI Emperor`_ mode and `systemd unit template`_ is what the distributors mostly offer their users, even if they differ in the way they implement both -modes and their defaults. Another point they might differ is the packaging of +modes and their defaults. Another point they might differ in is the packaging of plugins (if so, compare :ref:`install packages`) and what the default python interpreter is (python2 vs. python3). -Fedora starts a Emperor by default, while archlinux does not start any uwsgi -service by default. Worth to know; debian (ubuntu) follow a complete different -approach. *debian*: your are familiar with the apache infrastructure? .. they -do similar for the uWSGI infrastructure (with less comfort), the folders are:: +While archlinux does not start a uWSGI service by default, Fedora (RHEL) starts +a Emperor in `Tyrant mode`_ by default (you should have read :ref:`uWSGI Tyrant +mode pitfalls`). Worth to know; debian (ubuntu) follow a complete different +approach, read see :ref:`Debian's uWSGI layout`. + +.. _Debian's uWSGI layout: + +Debian's uWSGI layout +--------------------- + +.. _uwsgi.README.Debian: + https://salsa.debian.org/uwsgi-team/uwsgi/-/raw/debian/latest/debian/uwsgi.README.Debian + +Be aware, Debian's uWSGI layout is quite different from the standard uWSGI +configuration. Your are familiar with :ref:`Debian's Apache layout`? .. they do a +similar thing for the uWSGI infrastructure. The folders are:: /etc/uwsgi/apps-available/ /etc/uwsgi/apps-enabled/ @@ -82,29 +108,52 @@ The `uwsgi ini file`_ is enabled by a symbolic link:: ln -s /etc/uwsgi/apps-available/searxng.ini /etc/uwsgi/apps-enabled/ -From debian's documentation (``/usr/share/doc/uwsgi/README.Debian.gz``): You -could control specific instance(s) by issuing:: +More details can be found in the uwsgi.README.Debian_ +(``/usr/share/doc/uwsgi/README.Debian.gz``). Some commands you should know on +Debian: - service uwsgi <command> <confname> <confname> ... +.. code:: none - sudo -H service uwsgi start searx - sudo -H service uwsgi stop searx + Commands recognized by init.d script + ==================================== -My experience is, that this command is a bit buggy. + You can issue to init.d script following commands: + * start | starts daemon + * stop | stops daemon + * reload | sends to daemon SIGHUP signal + * force-reload | sends to daemon SIGTERM signal + * restart | issues 'stop', then 'start' commands + * status | shows status of daemon instance (running/not running) -.. _uwsgi configuration: + 'status' command must be issued with exactly one argument: '<confname>'. -Alltogether -=========== + Controlling specific instances of uWSGI + ======================================= -Create the configuration ini-file according to your distribution (see below) and -restart the uwsgi application. + You could control specific instance(s) by issuing: + + SYSTEMCTL_SKIP_REDIRECT=1 service uwsgi <command> <confname> <confname>... + + where: + * <command> is one of 'start', 'stop' etc. + * <confname> is the name of configuration file (without extension) + + For example, this is how instance for /etc/uwsgi/apps-enabled/hello.xml is + started: + + SYSTEMCTL_SKIP_REDIRECT=1 service uwsgi start hello + + +.. _uWSGI maintenance: + +uWSGI maintenance +================= .. tabs:: .. group-tab:: Ubuntu / debian - .. kernel-include:: $DOCS_BUILD/includes/searx.rst + .. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START searxng uwsgi-description ubuntu-20.04 :end-before: END searxng uwsgi-description ubuntu-20.04 @@ -112,7 +161,7 @@ restart the uwsgi application. .. group-tab:: Arch Linux - .. kernel-include:: $DOCS_BUILD/includes/searx.rst + .. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START searxng uwsgi-description arch :end-before: END searxng uwsgi-description arch @@ -120,16 +169,28 @@ restart the uwsgi application. .. group-tab:: Fedora / RHEL - .. kernel-include:: $DOCS_BUILD/includes/searx.rst + .. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START searxng uwsgi-description fedora :end-before: END searxng uwsgi-description fedora +.. _uwsgi setup: + +uWSGI setup +=========== + +Create the configuration ini-file according to your distribution and restart the +uwsgi application. As shown below, the :ref:`installation scripts` installs by +default: + +- a uWSGI setup that listens on a socket and +- enables :ref:`cache busting <static_use_hash>`. + .. tabs:: .. group-tab:: Ubuntu / debian - .. kernel-include:: $DOCS_BUILD/includes/searx.rst + .. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START searxng uwsgi-appini ubuntu-20.04 :end-before: END searxng uwsgi-appini ubuntu-20.04 @@ -137,7 +198,7 @@ restart the uwsgi application. .. group-tab:: Arch Linux - .. kernel-include:: $DOCS_BUILD/includes/searx.rst + .. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START searxng uwsgi-appini arch :end-before: END searxng uwsgi-appini arch @@ -145,6 +206,63 @@ restart the uwsgi application. .. group-tab:: Fedora / RHEL - .. kernel-include:: $DOCS_BUILD/includes/searx.rst + .. kernel-include:: $DOCS_BUILD/includes/searxng.rst :start-after: START searxng uwsgi-appini fedora :end-before: END searxng uwsgi-appini fedora + + +.. _uWSGI Tyrant mode pitfalls: + +Pitfalls of the Tyrant mode +=========================== + +The implementation of the process owners and groups in the `Tyrant mode`_ is +somewhat unusual and requires special consideration. In `Tyrant mode`_ mode the +Emperor will run the vassal using the UID/GID of the vassal configuration file +(user and group of the app ``.ini`` file). + +.. _#2099@uWSGI: https://github.com/unbit/uwsgi/issues/2099 +.. _#752@uWSGI: https://github.com/unbit/uwsgi/pull/752 +.. _#2425uWSGI: https://github.com/unbit/uwsgi/issues/2425 + +Without option ``emperor-tyrant-initgroups=true`` in ``/etc/uwsgi.ini`` the +process won't get the additional groups, but this option is not available in +2.0.x branch (see `#2099@uWSGI`_) the feature `#752@uWSGI`_ has been merged (on +Oct. 2014) to the master branch of uWSGI but had never been released; the last +major release is from Dec. 2013, since the there had been only bugfix releases +(see `#2425uWSGI`_). To shorten up: + + **In Tyrant mode, there is no way to get additional groups, and the uWSGI + process misses additional permissions that may be needed.** + +For example on Fedora (RHEL): If you try to install a redis DB with socket +communication and you want to connect to it from the SearXNG uWSGI, you will see a +*Permission denied* in the log of your instance:: + + ERROR:searx.shared.redis: [searxng (993)] can't connect redis DB ... + ERROR:searx.shared.redis: Error 13 connecting to unix socket: /usr/local/searxng-redis/run/redis.sock. Permission denied. + ERROR:searx.plugins.limiter: init limiter DB failed!!! + +Even if your *searxng* user of the uWSGI process is added to additional groups +to give access to the socket from the redis DB:: + + $ groups searxng + searxng : searxng searxng-redis + +To see the effective groups of the uwsgi process, you have to look at the status +of the process, by example:: + + $ ps -aef | grep '/usr/sbin/uwsgi --ini searxng.ini' + searxng 93 92 0 12:43 ? 00:00:00 /usr/sbin/uwsgi --ini searxng.ini + searxng 186 93 0 12:44 ? 00:00:01 /usr/sbin/uwsgi --ini searxng.ini + +Here you can see that the additional "Groups" of PID 186 are unset (missing gid +of ``searxng-redis``):: + + $ cat /proc/186/task/186/status + ... + Uid: 993 993 993 993 + Gid: 993 993 993 993 + FDSize: 128 + Groups: + ... diff --git a/docs/admin/installation.rst b/docs/admin/installation.rst index 91f82e504..cae51be63 100644 --- a/docs/admin/installation.rst +++ b/docs/admin/installation.rst @@ -4,109 +4,19 @@ Installation ============ -.. sidebar:: info - - :ref:`installation switch2ng` - *You're spoilt for choice*, choose your preferred method of installation. - :ref:`installation docker` - :ref:`installation scripts` - :ref:`installation basic` -The :ref:`installation basic` is good enough for intranet usage and it is a -excellent illustration of *how a SearXNG instance is build up*. If you place your -instance public to the internet you should really consider to install a -:ref:`filtron reverse proxy <filtron.sh>` and for privacy a :ref:`result proxy -<morty.sh>` is mandatory. - -Therefore, if you do not have any special preferences, its recommend to use the -:ref:`installation docker` or the `Installation scripts`_ from our :ref:`tooling -box <toolboxing>` as described below. - -.. _installation scripts: - -Installation scripts -==================== - -.. sidebar:: Update OS first! - - To avoid unwanted side effects, update your OS before installing SearXNG. - -The following will install a setup as shown in :ref:`architecture`. First you -need to get a clone. The clone is only needed for the installation procedure -and some maintenance tasks (alternatively you can create your own fork). - -For the installation procedure, use a *sudoer* login to run the scripts. If you -install from ``root``, take into account that the scripts are creating a -``searx``, a ``filtron`` and a ``morty`` user. In the installation procedure -these new created users do need read access to the clone of searx, which is not -the case if you clone into a folder below ``/root``. - -.. code:: bash - - $ cd ~/Downloads - $ git clone https://github.com/searxng/searxng.git searxng - $ cd searxng - -.. sidebar:: further read - - - :ref:`toolboxing` - - :ref:`update searxng` - - :ref:`inspect searxng` - -**Install** :ref:`SearXNG service <searx.sh>` - -This installs SearXNG as described in :ref:`installation basic`. - -.. code:: bash - - $ sudo -H ./utils/searx.sh install all - -**Install** :ref:`filtron reverse proxy <filtron.sh>` - -.. code:: bash - - $ sudo -H ./utils/filtron.sh install all - -**Install** :ref:`result proxy <morty.sh>` - -.. code:: bash - - $ sudo -H ./utils/morty.sh install all - -If all services are running fine, you can add it to your HTTP server: - -**Install** HTTP - -- :ref:`installation apache` -- :ref:`installation nginx` - -**Install** :ref:`external plugins <dev plugin>` - -Use SearXNG's ``shell`` to install external plugins. In the example below we -install the SearXNG plugins from **The Green Web Foundation** `[ref] -<https://www.thegreenwebfoundation.org/news/searching-the-green-web-with-searx/>`__: - -.. code:: bash - - $ sudo -H ./utils/searx.sh shell - // exit with [CTRL-D] - (searx-pyenv) searx@ryzen:~$ pip install git+https://github.com/return42/tgwf-searx-plugins - -In the :ref:`settings.yml` activate the ``plugins:`` section and add module -``only_show_green_results`` from tgwf-searx-plugins. - -.. code:: yaml - - plugins: - - only_show_green_results - -.. _git stash: https://git-scm.com/docs/git-stash +The :ref:`installation basic` is an excellent illustration of *how a SearXNG +instance is build up* (see :ref:`architecture uWSGI`). If you do not have any +special preferences, its recommend to use the :ref:`installation docker` or the +:ref:`installation scripts`. -.. tip:: +.. attention:: - About script's installation options have a look at chapter :ref:`toolboxing - setup`. How to brand your instance see chapter :ref:`settings global`. To - *stash* your instance's setup, `git stash`_ your clone's :origin:`.config.sh` - file . + SearXNG is growing rapidly, you should regularly read our :ref:`migrate and + stay tuned` section. If you want to upgrade an existing instance or migrate + from searx to SearXNG, you should read this section first! diff --git a/docs/admin/morty.rst b/docs/admin/morty.rst deleted file mode 100644 index b6bd2ea56..000000000 --- a/docs/admin/morty.rst +++ /dev/null @@ -1,40 +0,0 @@ - -.. _searxng morty: - -========================= -How to setup result proxy -========================= - -.. sidebar:: further reading - - - :ref:`morty.sh` - -.. _morty: https://github.com/asciimoo/morty -.. _morty's README: https://github.com/asciimoo/morty - -By default SearXNG can only act as an image proxy for result images, but it is -possible to proxify all the result URLs with an external service, morty_. - -To use this feature, morty has to be installed and activated in SearXNG's -``settings.yml``. Add the following snippet to your ``settings.yml`` and -restart searx: - -.. code:: yaml - - result_proxy: - url : http://127.0.0.1:3000/ - key : !!binary "insert_your_morty_proxy_key_here" - -Note that the example above (``http://127.0.0.1:3000``) is only for single-user -instances without a HTTP proxy. If your morty service is public, the url is the -address of the reverse proxy (e.g ``https://example.org/morty``). - -For more information about *result proxy* have a look at *"SearXNG via filtron -plus morty"* in the :ref:`nginx <nginx searxng via filtron plus morty>` and -:ref:`apache <apache searxng via filtron plus morty>` sections. - -``url`` - Is the address of the running morty service. - -``key`` - Is an optional argument, see `morty's README`_ for more information. diff --git a/docs/admin/update-searxng.rst b/docs/admin/update-searxng.rst index 4c541bf57..15735b0ba 100644 --- a/docs/admin/update-searxng.rst +++ b/docs/admin/update-searxng.rst @@ -1,59 +1,115 @@ +=================== +SearXNG maintenance +=================== + +.. sidebar:: further read + + - :ref:`toolboxing` + - :ref:`uWSGI maintenance` + +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + .. _update searxng: -============= How to update ============= How to update depends on the :ref:`installation` method. If you have used the -:ref:`installation scripts`, use ``update`` command from the scripts. - -**Update** :ref:`SearXNG service <searx.sh>` +:ref:`installation scripts`, use the ``update`` command from the :ref:`searxng.sh` +script. .. code:: sh - sudo -H ./utils/searx.sh update searx + sudo -H ./utils/searxng.sh instance update + +.. _inspect searxng: -**Update** :ref:`filtron reverse proxy <filtron.sh>` +How to inspect & debug +====================== + +How to debug depends on the :ref:`installation` method. If you have used the +:ref:`installation scripts`, use the ``inspect`` command from the :ref:`searxng.sh` +script. .. code:: sh - sudo -H ./utils/filtron.sh update filtron + sudo -H ./utils/searxng.sh instance inspect -**Update** :ref:`result proxy <morty.sh>` +.. _migrate and stay tuned: -.. code:: bash +Migrate and stay tuned! +======================= - $ sudo -H ./utils/morty.sh update morty +.. sidebar:: info -.. _inspect searxng: + - :pull:`1332` + - :pull:`456` + - :pull:`A comment about rolling release <446#issuecomment-954730358>` -====================== -How to inspect & debug -====================== +SearXNG is a *rolling release*; each commit to the master branch is a release. +SearXNG is growing rapidly, the services and opportunities are change every now +and then, to name just a few: -.. sidebar:: further read +- Bot protection has been switched from filtron to SearXNG's :ref:`limiter + <limiter>`, this requires a :ref:`Redis <settings redis>` database. - - :ref:`toolboxing` - - :ref:`Makefile` +- The image proxy morty is no longer needed, it has been replaced by the + :ref:`image proxy <image_proxy>` from SearXNG. -How to debug depends on the :ref:`installation` method. If you have used the -:ref:`installation scripts`, use ``inspect`` command from the scripts. +- To save bandwith :ref:`cache busting <static_use_hash>` has been implemented. + To get in use, the ``static-expires`` needs to be set in the :ref:`uwsgi + setup`. -**Inspect** :ref:`SearXNG service <searx.sh>` +To stay tuned and get in use of the new features, instance maintainers have to +update the SearXNG code regularly (see :ref:`update searxng`). As the above +examples show, this is not always enough, sometimes services have to be set up +or reconfigured and sometimes services that are no longer needed should be +uninstalled. -.. code:: sh +.. hint:: - sudo -H ./utils/searx.sh inspect service + First of all: SearXNG is installed by the script :ref:`searxng.sh`. If you + have old filtron, morty or searx setup you should consider complete + uninstall/reinstall. -**Inspect** :ref:`filtron reverse proxy <filtron.sh>` -.. code:: sh +remove obsolete services +------------------------ + +If your searx instance was installed *"Step by step"* or by the *"Installation +scripts"*, you need to undo the installation procedure completely. If you have +morty & filtron installed, it is recommended to uninstall these services also. +In case of scripts, to uninstall use the scripts from the origin you installed +searx from or try:: + + $ sudo -H ./utils/filtron.sh remove all + $ sudo -H ./utils/morty.sh remove all + $ sudo -H ./utils/searx.sh remove all + +.. hint:: + + If you are migrate from searx take into account that the ``.config.sh`` is no + longer used. - sudo -H ./utils/filtron.sh inspect service -**Inspect** :ref:`result proxy <morty.sh>` +Check after Installation +------------------------ -.. code:: bash +Once you have done your installation, you can run a SearXNG *check* procedure, +to see if there are some left overs. In this example there exists a *old* +``/etc/searx/settings.yml``:: - $ sudo -H ./utils/morty.sh inspect service + $ sudo -H ./utils/searxng.sh instance check + SearXNG checks + -------------- + ERROR: settings.yml in /etc/searx/ is deprecated, move file to folder /etc/searxng/ + INFO: [OK] (old) account 'searx' does not exists + INFO: [OK] (old) account 'filtron' does not exists + INFO: [OK] (old) account 'morty' does not exists + ... + INFO searx.shared : Use shared_simple implementation + INFO searx.shared.redis : connected redis DB --> default diff --git a/docs/build-templates/searx.rst b/docs/build-templates/searxng.rst index 626cce6bd..14b385468 100644 --- a/docs/build-templates/searx.rst +++ b/docs/build-templates/searxng.rst @@ -1,4 +1,4 @@ -.. template evaluated by: ./utils/searx.sh docs +.. template evaluated by: ./utils/searxng.sh searxng.doc.rst .. hint: all dollar-names are variables, dollar sign itself is quoted by: \\$ .. START distro-packages @@ -65,7 +65,8 @@ ${fedora_build} $ sudo -H useradd --shell /bin/bash --system \\ --home-dir \"$SERVICE_HOME\" \\ - --comment 'Privacy-respecting metasearch engine' $SERVICE_USER + --comment 'Privacy-respecting metasearch engine' \\ + $SERVICE_USER $ sudo -H mkdir \"$SERVICE_HOME\" $ sudo -H chown -R \"$SERVICE_GROUP:$SERVICE_GROUP\" \"$SERVICE_HOME\" @@ -81,7 +82,8 @@ ${fedora_build} .. code-block:: sh $ sudo -H -u ${SERVICE_USER} -i - (${SERVICE_USER})$ git clone \"$GIT_URL\" \"$SEARX_SRC\" + (${SERVICE_USER})$ git clone \"$GIT_URL\" \\ + \"$SEARXNG_SRC\" .. END clone searxng @@ -93,8 +95,9 @@ ${fedora_build} .. code-block:: sh - (${SERVICE_USER})$ python3 -m venv \"${SEARX_PYENV}\" - (${SERVICE_USER})$ echo \". ${SEARX_PYENV}/bin/activate\" >> \"$SERVICE_HOME/.profile\" + (${SERVICE_USER})$ python3 -m venv \"${SEARXNG_PYENV}\" + (${SERVICE_USER})$ echo \". ${SEARXNG_PYENV}/bin/activate\" \\ + >> \"$SERVICE_HOME/.profile\" .. END create virtualenv @@ -109,7 +112,7 @@ ${fedora_build} $ sudo -H -u ${SERVICE_USER} -i (${SERVICE_USER})$ command -v python && python --version - $SEARX_PYENV/bin/python + $SEARXNG_PYENV/bin/python Python 3.8.1 # update pip's boilerplate .. @@ -119,7 +122,7 @@ ${fedora_build} pip install -U pyyaml # jump to SearXNG's working tree and install SearXNG into virtualenv - (${SERVICE_USER})$ cd \"$SEARX_SRC\" + (${SERVICE_USER})$ cd \"$SEARXNG_SRC\" (${SERVICE_USER})$ pip install -e . @@ -134,24 +137,15 @@ ${fedora_build} .. code-block:: sh $ sudo -H mkdir -p \"$(dirname ${SEARXNG_SETTINGS_PATH})\" - $ sudo -H cp \"$SEARX_SRC/utils/templates/etc/searxng/settings.yml\" \\ + $ sudo -H cp \"$SEARXNG_SRC/utils/templates/etc/searxng/settings.yml\" \\ \"${SEARXNG_SETTINGS_PATH}\" - .. group-tab:: searx/settings.yml - - .. code-block:: sh - - $ sudo -H mkdir -p \"$(dirname ${SEARXNG_SETTINGS_PATH})\" - $ sudo -H cp \"$SEARX_SRC/searx/settings.yml\" \\ - \"${SEARXNG_SETTINGS_PATH}\" - -.. tabs:: - .. group-tab:: minimal setup .. code-block:: sh - $ sudo -H sed -i -e \"s/ultrasecretkey/\$(openssl rand -hex 16)/g\" \"$SEARXNG_SETTINGS_PATH\" + $ sudo -H sed -i -e \"s/ultrasecretkey/\$(openssl rand -hex 16)/g\" \\ + \"$SEARXNG_SETTINGS_PATH\" .. END searxng config @@ -168,14 +162,14 @@ ${fedora_build} # start webapp $ sudo -H -u ${SERVICE_USER} -i - (${SERVICE_USER})$ cd ${SEARX_SRC} + (${SERVICE_USER})$ cd ${SEARXNG_SRC} (${SERVICE_USER})$ export SEARXNG_SETTINGS_PATH=\"${SEARXNG_SETTINGS_PATH}\" (${SERVICE_USER})$ python searx/webapp.py # disable debug $ sudo -H sed -i -e \"s/debug : True/debug : False/g\" \"$SEARXNG_SETTINGS_PATH\" -Open WEB browser and visit http://$SEARX_INTERNAL_HTTP . If you are inside a +Open WEB browser and visit http://$SEARXNG_INTERNAL_HTTP . If you are inside a container or in a script, test with curl: .. tabs:: @@ -184,13 +178,13 @@ container or in a script, test with curl: .. code-block:: sh - $ xdg-open http://$SEARX_INTERNAL_HTTP + $ xdg-open http://$SEARXNG_INTERNAL_HTTP .. group-tab:: curl .. code-block:: none - $ curl --location --verbose --head --insecure $SEARX_INTERNAL_HTTP + $ curl --location --verbose --head --insecure $SEARXNG_INTERNAL_HTTP * Trying 127.0.0.1:8888... * TCP_NODELAY set diff --git a/docs/conf.py b/docs/conf.py index 44d1c2ad9..8e0c3ab1b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -195,5 +195,5 @@ html_show_sourcelink = True # LaTeX ---------------------------------------------------------------- latex_documents = [ - (master_doc, "searx-{}.tex".format(VERSION_STRING), html_title, author, "manual") + (master_doc, "searxng-{}.tex".format(VERSION_STRING), html_title, author, "manual") ] diff --git a/docs/dev/engine_overview.rst b/docs/dev/engine_overview.rst index 439875491..e950ae667 100644 --- a/docs/dev/engine_overview.rst +++ b/docs/dev/engine_overview.rst @@ -66,11 +66,11 @@ For a more detailed description, see :ref:`settings engine` in the :ref:`settin .. table:: Common options in the engine setup (``settings.yml``) :width: 100% - ======================= =========== =============================================== + ======================= =========== ================================================== argument type information - ======================= =========== =============================================== + ======================= =========== ================================================== name string name of search-engine - engine string name of searx-engine (filename without ``.py``) + engine string name of searxng-engine (file name without ``.py``) enable_http bool enable HTTP (by default only HTTPS is enabled). shortcut string shortcut of search-engine timeout string specific timeout for search-engine @@ -78,7 +78,7 @@ For a more detailed description, see :ref:`settings engine` in the :ref:`settin proxies dict set proxies for a specific engine (e.g. ``proxies : {http: socks5://proxy:port, https: socks5://proxy:port}``) - ======================= =========== =============================================== + ======================= =========== ================================================== .. _engine overrides: diff --git a/docs/dev/lxcdev.rst b/docs/dev/lxcdev.rst index 597f99925..6195a575d 100644 --- a/docs/dev/lxcdev.rst +++ b/docs/dev/lxcdev.rst @@ -45,9 +45,7 @@ be set on a *production* system. The scripts from :ref:`searx_utils` can divide in those to install and maintain software: -- :ref:`searx.sh` -- :ref:`filtron.sh` -- :ref:`morty.sh` +- :ref:`searxng.sh` and the script :ref:`lxc.sh`, with we can scale our installation, maintenance or even development tasks over a stack of isolated containers / what we call the: @@ -73,7 +71,7 @@ once: .. group-tab:: desktop - .. code:: sh + .. code:: bash $ snap install lxd $ lxd init --auto @@ -85,28 +83,28 @@ fork: .. group-tab:: desktop - .. code:: sh + .. code:: bash $ cd ~/Downloads $ git clone https://github.com/searxng/searxng.git searxng $ cd searxng -The :ref:`lxc-searx.env` consists of several images, see ``export -LXC_SUITE=(...`` near by :origin:`utils/lxc-searx.env#L19`. For this blog post +The :ref:`lxc-searxng.env` consists of several images, see ``export +LXC_SUITE=(...`` near by :origin:`utils/lxc-searxng.env#L19`. For this blog post we exercise on a archlinux_ image. The container of this image is named -``searx-archlinux``. Lets build the container, but be sure that this container +``searxng-archlinux``. Lets build the container, but be sure that this container does not already exists, so first lets remove possible old one: .. tabs:: .. group-tab:: desktop - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh remove searx-archlinux - $ sudo -H ./utils/lxc.sh build searx-archlinux + $ sudo -H ./utils/lxc.sh remove searxng-archlinux + $ sudo -H ./utils/lxc.sh build searxng-archlinux -.. sidebar:: The ``searx-archlinux`` container +.. sidebar:: The ``searxng-archlinux`` container is the base of all our exercises here. @@ -117,9 +115,9 @@ In this container we install all services :ref:`including searx, morty & filtron .. group-tab:: desktop - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh install suite searx-archlinux + $ sudo -H ./utils/lxc.sh install suite searxng-archlinux To proxy HTTP from filtron and morty in the container to the outside of the container, install nginx into the container. Once for the bot blocker filtron: @@ -128,9 +126,9 @@ container, install nginx into the container. Once for the bot blocker filtron: .. group-tab:: desktop - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \ ./utils/filtron.sh nginx install ... INFO: got 429 from http://10.174.184.156/searx @@ -141,9 +139,9 @@ and once for the content sanitizer (content proxy morty): .. group-tab:: desktop - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \ ./utils/morty.sh nginx install ... INFO: got 200 from http://10.174.184.156/morty/ @@ -154,7 +152,7 @@ and once for the content sanitizer (content proxy morty): blocker (filtron) and WEB content sanitizer (content proxy morty), both are needed for a *privacy protecting* search engine. -On your system, the IP of your ``searx-archlinux`` container differs from +On your system, the IP of your ``searxng-archlinux`` container differs from http://10.174.184.156/searx, just open the URL reported in your installation protocol in your WEB browser from the desktop to test the instance from outside of the container. @@ -169,27 +167,27 @@ In containers, work as usual Usually you open a root-bash using ``sudo -H bash``. In case of LXC containers open the root-bash in the container using ``./utils/lxc.sh cmd -searx-archlinux``: +searxng-archlinux``: .. tabs:: .. group-tab:: desktop - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux bash - INFO: [searx-archlinux] bash - [root@searx-archlinux searx]# pwd + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux bash + INFO: [searxng-archlinux] bash + [root@searxng-archlinux searx]# pwd /share/searxng -The prompt ``[root@searx-archlinux ...]`` signals, that you are the root user in -the searx-container. To debug the running SearXNG instance use: +The prompt ``[root@searxng-archlinux ...]`` signals, that you are the root user in +the searxng-container. To debug the running SearXNG instance use: .. tabs:: - .. group-tab:: root@searx-archlinux + .. group-tab:: root@searxng-archlinux - .. code:: sh + .. code:: bash $ ./utils/searx.sh inspect service ... @@ -202,56 +200,42 @@ above. You can stop monitoring using ``CTRL-C``, this also disables the *"debug option"* in SearXNG's settings file and restarts the SearXNG uwsgi application. To debug services from filtron and morty analogous use: -.. tabs:: - - .. group-tab:: root@searx-archlinux - - .. code:: sh - - $ ./utils/filtron.sh inspect service - $ ./utils/morty.sh inspect service - -Another point we have to notice is that each service (:ref:`SearXNG <searx.sh>`, -:ref:`filtron <filtron.sh>` and :ref:`morty <morty.sh>`) runs under dedicated -system user account with the same name (compare :ref:`create searxng user`). To -get a shell from theses accounts, simply call one of the scripts: +Another point we have to notice is that the service (:ref:`SearXNG <searxng.sh>` +runs under dedicated system user account with the same name (compare +:ref:`create searxng user`). To get a shell from theses accounts, simply call: .. tabs:: - .. group-tab:: root@searx-archlinux + .. group-tab:: root@searxng-archlinux - .. code:: sh + .. code:: bash - $ ./utils/searx.sh shell - $ ./utils/filtron.sh shell - $ ./utils/morty.sh shell + $ ./utils/searxng.sh instance cmd bash -To get in touch, open a shell from the service user (searx@searx-archlinux): +To get in touch, open a shell from the service user (searxng@searxng-archlinux): .. tabs:: .. group-tab:: desktop - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ - ./utils/searx.sh shell - // exit with [CTRL-D] - (searx-pyenv) [searx@searx-archlinux ~]$ ... + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux ./utils/searxng.sh instance cmd bash + INFO: [searxng-archlinux] ./utils/searxng.sh instance cmd bash + [searxng@searxng-archlinux ~]$ -The prompt ``[searx@searx-archlinux]`` signals that you are logged in as system -user ``searx`` in the ``searx-archlinux`` container and the python *virtualenv* -``(searx-pyenv)`` environment is activated. +The prompt ``[searxng@searxng-archlinux]`` signals that you are logged in as system +user ``searx`` in the ``searxng-archlinux`` container and the python *virtualenv* +``(searxng-pyenv)`` environment is activated. .. tabs:: - .. group-tab:: searx@searx-archlinux - - .. code:: sh + .. group-tab:: searxng@searxng-archlinux - (searx-pyenv) [searx@searx-archlinux ~]$ pwd - /usr/local/searx + .. code:: bash + (searxng-pyenv) [searxng@searxng-archlinux ~]$ pwd + /usr/local/searxng Wrap production into developer suite @@ -262,23 +246,22 @@ from a LXC container (which is quite ready for production) into a developer suite. For this, we have to keep an eye on the :ref:`installation basic`: - SearXNG setup in: ``/etc/searxng/settings.yml`` -- SearXNG user's home: ``/usr/local/searx`` -- virtualenv in: ``/usr/local/searx/searx-pyenv`` -- SearXNG software in: ``/usr/local/searx/searx-src`` +- SearXNG user's home: ``/usr/local/searxng`` +- virtualenv in: ``/usr/local/searxng/searxng-pyenv`` +- SearXNG software in: ``/usr/local/searxng/searxng-src`` -With the use of the :ref:`searx.sh` the SearXNG service was installed as +With the use of the :ref:`searxng.sh` the SearXNG service was installed as :ref:`uWSGI application <searxng uwsgi>`. To maintain this service, we can use -``systemctl`` (compare :ref:`service architectures on distributions <uwsgi -configuration>`). +``systemctl`` (compare :ref:`uWSGI maintenance`). .. tabs:: .. group-tab:: desktop - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ - systemctl stop uwsgi@searx + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \ + systemctl stop uwsgi@searxng With the command above, we stopped the SearXNG uWSGI-App in the archlinux container. @@ -291,29 +274,29 @@ least you should attend the settings of ``uid``, ``chdir``, ``env`` and env = SEARXNG_SETTINGS_PATH=/etc/searxng/settings.yml http = 127.0.0.1:8888 - chdir = /usr/local/searx/searx-src/searx - virtualenv = /usr/local/searx/searx-pyenv - pythonpath = /usr/local/searx/searx-src + chdir = /usr/local/searxng/searxng-src/searx + virtualenv = /usr/local/searxng/searxng-pyenv + pythonpath = /usr/local/searxng/searxng-src If you have read the :ref:`"Good to know section" <lxc.sh>` you remember, that each container shares the root folder of the repository and the command ``utils/lxc.sh cmd`` handles relative path names **transparent**. To wrap the SearXNG installation into a developer one, we simple have to create a smylink to the **transparent** reposetory from the desktop. Now lets replace the -repository at ``searx-src`` in the container with the working tree from outside +repository at ``searxng-src`` in the container with the working tree from outside of the container: .. tabs:: .. group-tab:: container becomes a developer suite - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ - mv /usr/local/searx/searx-src /usr/local/searx/searx-src.old + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \ + mv /usr/local/searxng/searxng-src /usr/local/searxng/searxng-src.old - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ - ln -s /share/searx/ /usr/local/searx/searx-src + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \ + ln -s /share/searx/ /usr/local/searxng/searxng-src Now we can develop as usual in the working tree of our desktop system. Every time the software was changed, you have to restart the SearXNG service (in the @@ -323,9 +306,9 @@ conatiner): .. group-tab:: desktop - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \ systemctl restart uwsgi@searx @@ -338,30 +321,30 @@ daily usage: To *inspect* the SearXNG instance (already described above): - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \ ./utils/searx.sh inspect service Run :ref:`makefile`, e.g. to test inside the container: - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \ make test To install all prerequisites needed for a :ref:`buildhosts`: - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ - ./utils/searx.sh install buildhost + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \ + ./utils/searxng.sh install buildhost To build the docs on a buildhost :ref:`buildhosts`: - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh cmd searx-archlinux \ + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux \ make docs.html .. _lxcdev summary: @@ -371,18 +354,18 @@ Summary We build up a fully functional SearXNG suite in a archlinux container: -.. code:: sh +.. code:: bash - $ sudo -H ./utils/lxc.sh install suite searx-archlinux + $ sudo -H ./utils/lxc.sh install suite searxng-archlinux To access HTTP from the desktop we installed nginx for the services inside the conatiner: .. tabs:: - .. group-tab:: [root@searx-archlinux] + .. group-tab:: [root@searxng-archlinux] - .. code:: sh + .. code:: bash $ ./utils/filtron.sh nginx install $ ./utils/morty.sh nginx install @@ -393,12 +376,12 @@ the container : .. tabs:: - .. group-tab:: [root@searx-archlinux] + .. group-tab:: [root@searxng-archlinux] - .. code:: sh + .. code:: bash - $ mv /usr/local/searx/searx-src /usr/local/searx/searx-src.old - $ ln -s /share/searx/ /usr/local/searx/searx-src + $ mv /usr/local/searxng/searxng-src /usr/local/searxng/searxng-src.old + $ ln -s /share/searx/ /usr/local/searxng/searxng-src $ systemctl restart uwsgi@searx To get information about the searxNG suite in the archlinux container we can @@ -408,13 +391,13 @@ use: .. group-tab:: desktop - .. code:: sh + .. code:: bash - $ sudo -H ./utils/lxc.sh show suite searx-archlinux + $ sudo -H ./utils/lxc.sh show suite searxng-archlinux ... - [searx-archlinux] INFO: (eth0) filtron: http://10.174.184.156:4004/ http://10.174.184.156/searx - [searx-archlinux] INFO: (eth0) morty: http://10.174.184.156:3000/ - [searx-archlinux] INFO: (eth0) docs.live: http://10.174.184.156:8080/ - [searx-archlinux] INFO: (eth0) IPv6: http://[fd42:573b:e0b3:e97e:216:3eff:fea5:9b65] + [searxng-archlinux] INFO: (eth0) filtron: http://10.174.184.156:4004/ http://10.174.184.156/searx + [searxng-archlinux] INFO: (eth0) morty: http://10.174.184.156:3000/ + [searxng-archlinux] INFO: (eth0) docs.live: http://10.174.184.156:8080/ + [searxng-archlinux] INFO: (eth0) IPv6: http://[fd42:573b:e0b3:e97e:216:3eff:fea5:9b65] ... diff --git a/docs/dev/makefile.rst b/docs/dev/makefile.rst index ceb76bf10..68c708a85 100644 --- a/docs/dev/makefile.rst +++ b/docs/dev/makefile.rst @@ -40,7 +40,7 @@ We do no longer need to build up the virtualenv manually. Jump into your git working tree and release a ``make install`` to get a virtualenv with a *developer install* of SearXNG (:origin:`setup.py`). :: - $ cd ~/searx-clone + $ cd ~/searxng-clone $ make install PYENV [virtualenv] installing ./requirements*.txt into local/py3 ... @@ -288,27 +288,3 @@ To filter out HTTP redirects (3xx_):: https://news.google.com:443 "GET /search?q=computer&hl=en&lr=lang_en&ie=utf8&oe=utf8&ceid=US%3Aen&gl=US HTTP/1.1" 302 0 https://news.google.com:443 "GET /search?q=computer&hl=en-US&lr=lang_en&ie=utf8&oe=utf8&ceid=US:en&gl=US HTTP/1.1" 200 None -- - - -``make pybuild`` -================ - -.. _PyPi: https://pypi.org/ -.. _twine: https://twine.readthedocs.io/en/latest/ - -Build Python packages in ``./dist/py``:: - - $ make pybuild - ... - BUILD pybuild - running sdist - running egg_info - ... - running bdist_wheel - - $ ls ./dist - searx-0.18.0-py3-none-any.whl searx-0.18.0.tar.gz - -To upload packages to PyPi_, there is also a ``pypi.upload`` target (to test use -``pypi.upload.test``). Since you are not the owner of :pypi:`searx` you will -never need to upload. diff --git a/docs/dev/offline_engines.rst b/docs/dev/offline_engines.rst index 9320c6262..bfb2664f5 100644 --- a/docs/dev/offline_engines.rst +++ b/docs/dev/offline_engines.rst @@ -55,10 +55,10 @@ admins can install packages in advance. If there is a need to install additional packages in *Python's Virtual Environment* of your SearXNG instance you need to switch into the environment -(:ref:`searx-src`) first, for this you can use :ref:`searx.sh`:: +(:ref:`searxng-src`) first, for this you can use :ref:`searxng.sh`:: - $ sudo utils/searx.sh shell - (searx-pyenv)$ pip install ... + $ sudo utils/searxng.sh instance cmd bash + (searxng-pyenv)$ pip install ... Private engines (Security) diff --git a/docs/dev/plugins.rst b/docs/dev/plugins.rst index 44401e34f..36a443230 100644 --- a/docs/dev/plugins.rst +++ b/docs/dev/plugins.rst @@ -33,17 +33,26 @@ Example plugin External plugins ================ -External plugins are standard python modules implementing all the requirements of the standard plugins. -Plugins can be enabled by adding them to :ref:`settings.yml`'s ``plugins`` section. -Example external plugin can be found `here <https://github.com/asciimoo/searx_external_plugin_example>`_. +SearXNG supports *external plugins* / there is no need to install one, SearXNG +runs out of the box. But to demonstrate; in the example below we install the +SearXNG plugins from *The Green Web Foundation* `[ref] +<https://www.thegreenwebfoundation.org/news/searching-the-green-web-with-searx/>`__: -Register your plugin -==================== +.. code:: bash + + $ sudo utils/searxng.sh instance cmd bash + (searxng-pyenv)$ pip install git+https://github.com/return42/tgwf-searx-plugins + +In the :ref:`settings.yml` activate the ``plugins:`` section and add module +``only_show_green_results`` from ``tgwf-searx-plugins``. + +.. code:: yaml + + plugins: + ... + - only_show_green_results + ... -To enable your plugin register your plugin in -searx > plugin > __init__.py. -And at the bottom of the file add your plugin like. -``plugins.register(name_of_python_file)`` Plugin entry points =================== diff --git a/docs/dev/quickstart.rst b/docs/dev/quickstart.rst index db52a2d80..921384aab 100644 --- a/docs/dev/quickstart.rst +++ b/docs/dev/quickstart.rst @@ -10,7 +10,7 @@ Development Quickstart SearXNG loves developers, just clone and start hacking. All the rest is done for you simply by using :ref:`make <makefile>`. -.. code:: sh +.. code:: bash git clone https://github.com/searxng/searxng.git searxng @@ -27,21 +27,21 @@ to our ":ref:`how to contribute`" guideline. If you implement themes, you will need to setup a :ref:`make node.env` once: -.. code:: sh +.. code:: bash make node.env Before you call *make run* (2.), you need to compile the modified styles and JavaScript: -.. code:: sh +.. code:: bash make themes.all Alternatively you can also compile selective the theme you have modified, e.g. the *simple* theme. -.. code:: sh +.. code:: bash make themes.simple @@ -52,7 +52,7 @@ e.g. the *simple* theme. If you finished your *tests* you can start to commit your changes. To separate the modified source code from the build products first run: -.. code:: sh +.. code:: bash make static.build.restore @@ -60,13 +60,13 @@ This will restore the old build products and only your changes of the code remain in the working tree which can now be added & commited. When all sources are commited, you can commit the build products simply by: -.. code:: sh +.. code:: bash make static.build.commit Commiting the build products should be the last step, just before you send us your PR. There is also a make target to rewind this last build commit: -.. code:: sh +.. code:: bash make static.build.drop diff --git a/docs/utils/filtron.sh.rst b/docs/utils/filtron.sh.rst deleted file mode 100644 index 1f7302392..000000000 --- a/docs/utils/filtron.sh.rst +++ /dev/null @@ -1,80 +0,0 @@ - -.. _filtron.sh: - -==================== -``utils/filtron.sh`` -==================== - -.. sidebar:: further reading - - - :ref:`searxng filtron` - - :ref:`architecture` - - :ref:`installation` (:ref:`nginx <installation nginx>` & :ref:`apache - <installation apache>`) - -.. _Go: https://golang.org/ -.. _filtron: https://github.com/searxng/filtron -.. _filtron README: https://github.com/searxng/filtron/blob/master/README.md - -To simplify installation and maintenance of a filtron instance you can use the -script :origin:`utils/filtron.sh`. In most cases you will install filtron_ -simply by running the command: - -.. code:: bash - - sudo -H ./utils/filtron.sh install all - -The script adds a ``${SERVICE_USER}`` (default:``filtron``) and installs filtron_ -into this user account: - -#. Create a separated user account (``filtron``). -#. Download and install Go_ binary in user's $HOME (``~filtron``). -#. Install filtron with the package management from Go_ (``go get -v -u - github.com/searxng/filtron``) -#. Setup a proper rule configuration :origin:`[ref] - <utils/templates/etc/filtron/rules.json>` (``/etc/filtron/rules.json``). -#. Setup a systemd service unit :origin:`[ref] - <utils/templates/lib/systemd/system/filtron.service>` - (``/lib/systemd/system/filtron.service``). - - -Create user -=========== - -.. kernel-include:: $DOCS_BUILD/includes/filtron.rst - :start-after: START create user - :end-before: END create user - - -Install go -========== - -.. kernel-include:: $DOCS_BUILD/includes/filtron.rst - :start-after: START install go - :end-before: END install go - - -Install filtron -=============== - -Install :origin:`rules.json <utils/templates/etc/filtron/rules.json>` at -``/etc/filtron/rules.json`` (see :ref:`Sample configuration of filtron`) and -install filtron software and systemd unit: - -.. kernel-include:: $DOCS_BUILD/includes/filtron.rst - :start-after: START install filtron - :end-before: END install filtron - -.. kernel-include:: $DOCS_BUILD/includes/filtron.rst - :start-after: START install systemd unit - :end-before: END install systemd unit - -.. _filtron.sh overview: - -Overview -======== - -The ``--help`` output of the script is largely self-explanatory -(:ref:`toolboxing common`): - -.. program-output:: ../utils/filtron.sh --help diff --git a/docs/utils/index.rst b/docs/utils/index.rst index 2077bbb13..2da26ed12 100644 --- a/docs/utils/index.rst +++ b/docs/utils/index.rst @@ -1,52 +1,30 @@ .. _searx_utils: .. _toolboxing: -=================== -Admin's tooling box -=================== +================== +DevOps tooling box +================== -In the folder :origin:`utils/` we maintain some tools useful for administrators. +In the folder :origin:`utils/` we maintain some tools useful for administrators +and developers. .. toctree:: :maxdepth: 2 :caption: Contents - searx.sh - filtron.sh - morty.sh + searxng.sh lxc.sh -.. _toolboxing common: +Common command environments +=========================== -Common commands & environment -============================= - -Scripts to maintain services often dispose of common commands and environments. - -``shell`` : command - Opens a shell from the service user ``${SERVICE_USSR}``, very helpful for - troubleshooting. - -``inspect service`` : command - Shows status and log of the service, most often you have a option to enable - more verbose debug logs. Very helpful for debugging, but be careful not to - enable debugging in a production environment! +The scripts in our tooling box often dispose of common environments: ``FORCE_TIMEOUT`` : environment Sets timeout for interactive prompts. If you want to run a script in batch job, with defaults choices, set ``FORCE_TIMEOUT=0``. By example; to install a - reverse proxy for filtron on all containers of the :ref:`SearXNG suite - <lxc-searx.env>` use :: - - sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/filtron.sh apache install - -.. _toolboxing setup: - -Tooling box setup -================= - -The main setup is done in the :origin:`.config.sh` (read also :ref:`settings -global`). + SearXNG server and nginx proxy on all containers of the :ref:`SearXNG suite + <lxc-searxng.env>` use:: -.. literalinclude:: ../../.config.sh - :language: bash + sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/searxng.sh install all + sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/searxng.sh install nginx diff --git a/docs/utils/lxc.sh.rst b/docs/utils/lxc.sh.rst index f09b033f4..4308a12cc 100644 --- a/docs/utils/lxc.sh.rst +++ b/docs/utils/lxc.sh.rst @@ -23,7 +23,7 @@ With the use of *Linux Containers* (LXC_) we can scale our tasks over a stack of containers, what we call the: *lxc suite*. The *SearXNG suite* -(:origin:`lxc-searx.env <utils/lxc-searx.env>`) is loaded by default, every time +(:origin:`lxc-searxng.env <utils/lxc-searxng.env>`) is loaded by default, every time you start the ``lxc.sh`` script (*you do not need to care about*). Before you can start with containers, you need to install and initiate LXD_ @@ -49,7 +49,7 @@ help>`. If you do not want to build all containers, **you can build just one**:: - $ sudo -H ./utils/lxc.sh build searx-ubu1804 + $ sudo -H ./utils/lxc.sh build searxng-archlinux *Good to know ...* @@ -62,9 +62,9 @@ of:: In the containers, you can run what ever you want, e.g. to start a bash use:: - $ sudo -H ./utils/lxc.sh cmd searx-ubu1804 bash - INFO: [searx-ubu1804] bash - root@searx-ubu1804:/share/searx# + $ sudo -H ./utils/lxc.sh cmd searxng-archlinux bash + INFO: [searxng-archlinux] bash + [root@searxng-archlinux SearXNG]# If there comes the time you want to **get rid off all** the containers and **clean up local images** just type:: @@ -121,28 +121,26 @@ Install suite ============= To install the complete :ref:`SearXNG suite (includes searx, morty & filtron) -<lxc-searx.env>` into all LXC_ use:: +<lxc-searxng.env>` into all LXC_ use:: $ sudo -H ./utils/lxc.sh install suite -The command above installs a SearXNG suite (see :ref:`installation scripts`). To -get the IP (URL) of the filtron service in the containers use ``show suite`` +The command above installs a SearXNG suite (see :ref:`installation scripts`). +To :ref:`install a nginx <installation nginx>` reverse proxy (or alternatively +use :ref:`apache <installation apache>`):: + + sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/searxng.sh install nginx + +To get the IP (URL) of the SearXNG service in the containers use ``show suite`` command. To test instances from containers just open the URLs in your WEB-Browser:: - $ sudo ./utils/lxc.sh show suite | grep filtron - [searx-ubu1604] INFO: (eth0) filtron: http://n.n.n.246:4004/ http://n.n.n.246/searx - [searx-ubu1804] INFO: (eth0) filtron: http://n.n.n.147:4004/ http://n.n.n.147/searx - [searx-ubu1910] INFO: (eth0) filtron: http://n.n.n.140:4004/ http://n.n.n.140/searx - [searx-ubu2004] INFO: (eth0) filtron: http://n.n.n.18:4004/ http://n.n.n.18/searx - [searx-fedora31] INFO: (eth0) filtron: http://n.n.n.46:4004/ http://n.n.n.46/searx - [searx-archlinux] INFO: (eth0) filtron: http://n.n.n.32:4004/ http://n.n.n.32/searx - -To :ref:`install a nginx <installation nginx>` reverse proxy for filtron and -morty use (or alternatively use :ref:`apache <installation apache>`):: + $ sudo ./utils/lxc.sh show suite | grep SEARXNG_URL - sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/filtron.sh nginx install - sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/morty.sh nginx install + [searxng-ubu2110] SEARXNG_URL : http://n.n.n.147/searxng + [searxng-ubu2004] SEARXNG_URL : http://n.n.n.246/searxng + [searxnggfedora35] SEARXNG_URL : http://n.n.n.140/searxng + [searxng-archlinux] SEARXNG_URL : http://n.n.n.165/searxng Running commands @@ -152,8 +150,8 @@ Running commands :ref:`toolboxing`. By example: to setup a :ref:`buildhosts` and run the Makefile target ``test`` in the archlinux_ container:: - sudo -H ./utils/lxc.sh cmd searx-archlinux ./utils/searx.sh install buildhost - sudo -H ./utils/lxc.sh cmd searx-archlinux make test + sudo -H ./utils/lxc.sh cmd searxng-archlinux ./utils/searxng.sh install buildhost + sudo -H ./utils/lxc.sh cmd searxng-archlinux make test Setup SearXNG buildhost @@ -164,11 +162,11 @@ The installation procedure to set up a :ref:`build host<buildhosts>` takes its time. Installation in all containers will take more time (time for another cup of coffee).:: - sudo -H ./utils/lxc.sh cmd -- ./utils/searx.sh install buildhost + sudo -H ./utils/lxc.sh cmd -- ./utils/searxng.sh install buildhost To build (live) documentation inside a archlinux_ container:: - sudo -H ./utils/lxc.sh cmd searx-archlinux make docs.clean docs.live + sudo -H ./utils/lxc.sh cmd searxng-archlinux make docs.clean docs.live ... [I 200331 15:00:42 server:296] Serving on http://0.0.0.0:8080 @@ -176,7 +174,7 @@ To get IP of the container and the port number *live docs* is listening:: $ sudo ./utils/lxc.sh show suite | grep docs.live ... - [searx-archlinux] INFO: (eth0) docs.live: http://n.n.n.12:8080/ + [searxng-archlinux] INFO: (eth0) docs.live: http://n.n.n.12:8080/ .. _lxc.sh help: @@ -189,10 +187,10 @@ The ``--help`` output of the script is largely self-explanatory: .. program-output:: ../utils/lxc.sh --help -.. _lxc-searx.env: +.. _lxc-searxng.env: SearXNG suite ============= -.. literalinclude:: ../../utils/lxc-searx.env +.. literalinclude:: ../../utils/lxc-searxng.env :language: bash diff --git a/docs/utils/morty.sh.rst b/docs/utils/morty.sh.rst deleted file mode 100644 index 99103b1cb..000000000 --- a/docs/utils/morty.sh.rst +++ /dev/null @@ -1,80 +0,0 @@ - -.. _morty: https://github.com/asciimoo/morty -.. _morty's README: https://github.com/asciimoo/morty -.. _Go: https://golang.org/ - -.. _morty.sh: - -================== -``utils/morty.sh`` -================== - -.. sidebar:: further reading - - - :ref:`architecture` - - :ref:`installation` (:ref:`nginx <installation nginx>` & :ref:`apache - <installation apache>`) - - :ref:`searxng morty` - -To simplify installation and maintenance of a morty_ instance you can use the -script :origin:`utils/morty.sh`. In most cases you will install morty_ simply by -running the command: - -.. code:: bash - - sudo -H ./utils/morty.sh install all - -The script adds a ``${SERVICE_USER}`` (default:``morty``) and installs morty_ -into this user account: - -#. Create a separated user account (``morty``). -#. Download and install Go_ binary in user's $HOME (``~morty``). -#. Install morty_ with the package management from Go_ (``go get -v -u - github.com/asciimoo/morty``) -#. Setup a systemd service unit :origin:`[ref] - <utils/templates/lib/systemd/system/morty.service>` - (``/lib/systemd/system/morty.service``). - -.. hint:: - - To add morty to your SearXNG instance read chapter :ref:`searxng morty`. - -Create user -=========== - -.. kernel-include:: $DOCS_BUILD/includes/morty.rst - :start-after: START create user - :end-before: END create user - - -Install go -========== - -.. kernel-include:: $DOCS_BUILD/includes/morty.rst - :start-after: START install go - :end-before: END install go - - -Install morty -============= - -Install morty software and systemd unit: - -.. kernel-include:: $DOCS_BUILD/includes/morty.rst - :start-after: START install morty - :end-before: END install morty - -.. kernel-include:: $DOCS_BUILD/includes/morty.rst - :start-after: START install systemd unit - :end-before: END install systemd unit - -.. _morty.sh overview: - -Overview -======== - -The ``--help`` output of the script is largely self-explanatory -(:ref:`toolboxing common`): - -.. program-output:: ../utils/morty.sh --help - diff --git a/docs/utils/searx.sh.rst b/docs/utils/searx.sh.rst deleted file mode 100644 index 6d412d23d..000000000 --- a/docs/utils/searx.sh.rst +++ /dev/null @@ -1,39 +0,0 @@ - -.. _searx.sh: - -================== -``utils/searx.sh`` -================== - -.. sidebar:: further reading - - - :ref:`architecture` - - :ref:`installation` - - :ref:`installation nginx` - - :ref:`installation apache` - -To simplify installation and maintenance of a SearXNG instance you can use the -script :origin:`utils/searx.sh`. - -Install -======= - -In most cases you will install SearXNG simply by running the command: - -.. code:: bash - - sudo -H ./utils/searx.sh install all - -The script adds a ``${SERVICE_USER}`` (default:``searx``) and installs SearXNG -into this user account. The installation is described in chapter -:ref:`installation basic`. - -.. _intranet reverse proxy: - -Overview -======== - -The ``--help`` output of the script is largely self-explanatory -(:ref:`toolboxing common`): - -.. program-output:: ../utils/searx.sh --help diff --git a/docs/utils/searxng.sh.rst b/docs/utils/searxng.sh.rst new file mode 100644 index 000000000..f6578f7b1 --- /dev/null +++ b/docs/utils/searxng.sh.rst @@ -0,0 +1,36 @@ + +.. _searxng.sh: + +==================== +``utils/searxng.sh`` +==================== + +.. sidebar:: further reading + + - :ref:`architecture` + - :ref:`installation` + - :ref:`installation nginx` + - :ref:`installation apache` + +To simplify the installation and maintenance of a SearXNG instance you can use the +script :origin:`utils/searxng.sh`. + +Install +======= + +In most cases you will install SearXNG simply by running the command: + +.. code:: bash + + sudo -H ./utils/searx.sh install all + +The installation is described in chapter :ref:`installation basic`. + +.. _searxng.sh overview: + +Overview +======== + +The ``--help`` output of the script is largely self-explanatory: + +.. program-output:: ../utils/searxng.sh --help |