diff options
68 files changed, 7300 insertions, 537 deletions
diff --git a/.config.sh b/.config.sh new file mode 100644 index 000000000..4eff5f4c6 --- /dev/null +++ b/.config.sh @@ -0,0 +1,55 @@ +# -*- coding: utf-8; mode: sh -*- +# SPDX-License-Identifier: AGPL-3.0-or-later +# shellcheck shell=bash disable=SC2034 +# +# This environment is used by ./utils scripts like filtron.sh or searx.sh. The +# default values are *most flexible* and *best maintained*, you normally not +# need to change the defaults (except PUBLIC_URL). +# +# Before you change any value here you have to uninstall any previous +# installation. Further is it recommended to backup your changes simply by +# adding them to you local brand (git branch):: +# +# git add .config + +# The public URL of the searx instance: PUBLIC_URL="https://mydomain.xy/searx" +# The default is taken from ./utils/brand.env. + +PUBLIC_URL="${SEARX_URL}" + +if [[ ${PUBLIC_URL} == "https://searx.me" ]]; then + # hint: Linux containers do not have DNS entries, lets use IPs + PUBLIC_URL="http://$(primary_ip)/searx" +fi + +# searx.sh +# --------- + +# SEARX_INTERNAL_URL="127.0.0.1:8888" + +# Only change, if you maintain a searx brand in your searx fork. +# GIT_BRANCH="${GIT_BRANCH:-master}" + +# filtron.sh +# ---------- + +# FILTRON_API="127.0.0.1:4005" +# FILTRON_LISTEN="127.0.0.1:4004" +# FILTRON_TARGET="127.0.0.1:8888" + +# morty.sh +# -------- + +# morty listen address +# MORTY_LISTEN="127.0.0.1:3000" +# PUBLIC_URL_PATH_MORTY="/morty/" + +# system services +# --------------- + +# Common $HOME folder of the service accounts +# SERVICE_HOME_BASE="/usr/local" + +# **experimental**: Set SERVICE_USER to run all services by one account, but be +# aware that removing discrete components might conflict! +# SERVICE_USER=searx diff --git a/.gitignore b/.gitignore index 1c82be41b..e56a575ab 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # to sync with .dockerignore .coverage coverage/ +cache/ .installed.cfg engines.cfg env @@ -1,24 +1,31 @@ # -*- coding: utf-8; mode: makefile-gmake -*- +.DEFAULT_GOAL=help +# START Makefile setup export GIT_URL=https://github.com/asciimoo/searx +export GIT_BRANCH=master export SEARX_URL=https://searx.me export DOCS_URL=https://asciimoo.github.io/searx +# END Makefile setup + +include utils/makefile.include PYOBJECTS = searx DOC = docs PY_SETUP_EXTRAS ?= \[test\] -PYDIST=./dist/py -PYBUILD=./build/py - -include utils/makefile.include include utils/makefile.python include utils/makefile.sphinx all: clean install -PHONY += help -help: +PHONY += help-min help-all help + +help: help-min + @echo '' + @echo 'to get more help: make help-all' + +help-min: @echo ' test - run developer tests' @echo ' docs - build documentation' @echo ' docs-live - autobuild HTML documentation while editing' @@ -33,9 +40,18 @@ help: @echo ' docker - build Docker image' @echo ' node.env - download & install npm dependencies locally' @echo '' - @$(MAKE) -s -f utils/makefile.include make-help + @echo 'environment' + @echo ' SEARX_URL = $(SEARX_URL)' + @echo ' GIT_URL = $(GIT_URL)' + @echo ' DOCS_URL = $(DOCS_URL)' + @echo '' + @$(MAKE) -e -s make-help + +help-all: help-min + @echo '' + @$(MAKE) -e -s python-help @echo '' - @$(MAKE) -s -f utils/makefile.python python-help + @$(MAKE) -e -s docs-help PHONY += install install: buildenv pyenvinstall @@ -44,7 +60,7 @@ PHONY += uninstall uninstall: pyenvuninstall PHONY += clean -clean: pyclean node.clean test.clean +clean: pyclean docs-clean node.clean test.clean $(call cmd,common_clean) PHONY += run @@ -61,14 +77,24 @@ run: buildenv pyenvinstall # docs # ---- +sphinx-doc-prebuilds:: buildenv pyenvinstall prebuild-includes + PHONY += docs -docs: buildenv pyenvinstall sphinx-doc +docs: sphinx-doc-prebuilds sphinx-doc $(call cmd,sphinx,html,docs,docs) PHONY += docs-live -docs-live: buildenv pyenvinstall sphinx-live +docs-live: sphinx-doc-prebuilds sphinx-live $(call cmd,sphinx_autobuild,html,docs,docs) +PHONY += prebuild-includes +prebuild-includes: + $(Q)mkdir -p $(DOCS_BUILD)/includes + $(Q)./utils/searx.sh doc | cat > $(DOCS_BUILD)/includes/searx.rst + $(Q)./utils/filtron.sh doc | cat > $(DOCS_BUILD)/includes/filtron.rst + $(Q)./utils/morty.sh doc | cat > $(DOCS_BUILD)/includes/morty.rst + + $(GH_PAGES):: @echo "doc available at --> $(DOCS_URL)" @@ -94,12 +120,14 @@ useragents.update: pyenvinstall buildenv: $(Q)echo "build searx/brand.py" $(Q)echo "GIT_URL = '$(GIT_URL)'" > searx/brand.py + $(Q)echo "GIT_BRANCH = '$(GIT_BRANCH)'" >> searx/brand.py $(Q)echo "ISSUE_URL = 'https://github.com/asciimoo/searx/issues'" >> searx/brand.py $(Q)echo "SEARX_URL = '$(SEARX_URL)'" >> searx/brand.py $(Q)echo "DOCS_URL = '$(DOCS_URL)'" >> searx/brand.py $(Q)echo "PUBLIC_INSTANCES = 'https://searx.space'" >> searx/brand.py $(Q)echo "build utils/brand.env" $(Q)echo "export GIT_URL='$(GIT_URL)'" > utils/brand.env + $(Q)echo "export GIT_BRANCH='$(GIT_BRANCH)'" >> utils/brand.env $(Q)echo "export ISSUE_URL='https://github.com/asciimoo/searx/issues'" >> utils/brand.env $(Q)echo "export SEARX_URL='$(SEARX_URL)'" >> utils/brand.env $(Q)echo "export DOCS_URL='$(DOCS_URL)'" >> utils/brand.env @@ -182,8 +210,7 @@ gecko.driver: # test # ---- -PHONY += test test.pylint test.pep8 test.unit test.coverage test.robot - +PHONY += test test.sh test.pylint test.pep8 test.unit test.coverage test.robot test: buildenv test.pylint test.pep8 test.unit gecko.driver test.robot ifeq ($(PY),2) @@ -191,6 +218,7 @@ test.pylint: @echo "LINT skip liniting py2" else # TODO: balance linting with pylint + test.pylint: pyenvinstall $(call cmd,pylint,\ searx/preferences.py \ @@ -202,6 +230,17 @@ endif # E402 module level import not at top of file # W503 line break before binary operator +# ubu1604: uses shellcheck v0.3.7 (from 04/2015), no longer supported! +test.sh: + shellcheck -x -s bash utils/brand.env + shellcheck -x utils/lib.sh + shellcheck -x utils/filtron.sh + shellcheck -x utils/searx.sh + shellcheck -x utils/morty.sh + shellcheck -x utils/lxc.sh + shellcheck -x utils/lxc-searx.env + shellcheck -x .config.sh + test.pep8: pyenvinstall @echo "TEST pep8" $(Q)$(PY_ENV_ACT); pep8 --exclude=searx/static --max-line-length=120 --ignore "E402,W503" searx tests diff --git a/docs/_themes/searx/static/searx.css b/docs/_themes/searx/static/searx.css index d6a664f0f..0f2eff728 100644 --- a/docs/_themes/searx/static/searx.css +++ b/docs/_themes/searx/static/searx.css @@ -33,7 +33,7 @@ p.sidebar-title, .sidebar p { /* admonitions */ -div.admonition, div.topic { +div.admonition, div.topic, div.toctree-wrapper { background-color: #fafafa; margin: 8px 0px; padding: 1em; @@ -42,6 +42,16 @@ div.admonition, div.topic { border-right: none; border-bottom: none; border-left: 5pt solid #ccc; + list-style-type: disclosure-closed; +} + +div.toctree-wrapper p.caption { + font-weight: normal; + font-size: 24px; + margin: 0 0 10px 0; + padding: 0; + line-height: 1; + display: inline; } p.admonition-title:after { @@ -128,3 +138,32 @@ caption { caption-side: top; text-align: left; } + +/* bugs since sphinx 3.1 + +See sphinx-doc project, PR 7838 & 7484 with elementary patch to the basic CSS: + +- https://github.com/sphinx-doc/sphinx/issues/7838#issuecomment-646009605 +- https://github.com/sphinx-doc/sphinx/pull/7484#issuecomment-646058972 + +*/ + +li > p:first-child { + margin-top: 0; +} + +li > p:last-child { + margin-bottom: 0; +} + +div.admonition dl { + margin-bottom: 0; +} + +div.sidebar { + clear: none; +} + +div.admonition, div.topic, pre { + clear: none; +} diff --git a/docs/admin/arch_public.dot b/docs/admin/arch_public.dot index a46c96de3..5018225c3 100644 --- a/docs/admin/arch_public.dot +++ b/docs/admin/arch_public.dot @@ -4,11 +4,11 @@ digraph G { edge [fontname="Sans"]; browser [label="Browser", shape=Mdiamond]; - rp [label="Reverse Proxy", href="url to configure reverse proxy"]; - filtron [label="Filtron", href="https://github.com/asciimoo/filtron"]; - morty [label="Morty", href="https://github.com/asciimoo/morty"]; + rp [label="Reverse Proxy", href="https://asciimoo.github.io/searx/utils/filtron.sh.html#public-reverse-proxy"]; + filtron [label="Filtron", href="https://asciimoo.github.io/searx/utils/filtron.sh.html"]; + morty [label="Morty", href="https://asciimoo.github.io/searx/utils/morty.sh.html"]; static [label="Static files", href="url to configure static files"]; - uwsgi [label="uwsgi", href="url to configure uwsgi"] + uwsgi [label="uwsgi", href="https://asciimoo.github.io/searx/utils/searx.sh.html"] searx1 [label="Searx #1"]; searx2 [label="Searx #2"]; searx3 [label="Searx #3"]; diff --git a/docs/admin/architecture.rst b/docs/admin/architecture.rst index 7064a294b..464e765eb 100644 --- a/docs/admin/architecture.rst +++ b/docs/admin/architecture.rst @@ -4,17 +4,21 @@ Architecture ============ -.. sidebar:: Needs work! +.. sidebar:: Further reading - This article needs some work / Searx is a collaborative effort. If you have - any contribution, feel welcome to send us your :pull:`PR <../pulls>`, see - :ref:`how to contribute`. + - Reverse Proxy: :ref:`Apache <apache searx site>` & :ref:`nginx <nginx searx + site>` + - Filtron: :ref:`searx filtron` + - Morty: :ref:`searx morty` + - uWSGI: :ref:`searx uwsgi` + - Searx: :ref:`installation basic` Herein you will find some hints and suggestions about typical architectures of searx infrastructures. We start with a contribution from :pull:`@dalf <1776#issuecomment-567917320>`. -It shows a *reference* setup for public searx instances. +It shows a *reference* setup for public searx instances which can build up and +maintained by the scripts from our :ref:`toolboxing`. .. _arch public: diff --git a/docs/admin/buildhosts.rst b/docs/admin/buildhosts.rst index 5260da033..a727d25b9 100644 --- a/docs/admin/buildhosts.rst +++ b/docs/admin/buildhosts.rst @@ -9,8 +9,27 @@ Buildhosts If you have any contribution send us your :pull:`PR <../pulls>`, see :ref:`how to contribute`. +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + To get best results from build, its recommend to install additional packages -on build hosts. +on build hosts (see :ref:`searx.sh`).:: + + sudo -H ./utils/searx.sh install buildhost + +This will install packages needed by searx: + +.. kernel-include:: $DOCS_BUILD/includes/searx.rst + :start-after: START distro-packages + :end-before: END distro-packages + +and packages needed to build docuemtation and run tests: + +.. kernel-include:: $DOCS_BUILD/includes/searx.rst + :start-after: START build-packages + :end-before: END build-packages .. _docs build: @@ -35,8 +54,17 @@ processing additional packages are needed. The XeTeX_ needed not only for PDF creation, its also needed for :ref:`math` when HTML output is build. To be able to do :ref:`sphinx:math-support` without CDNs, the math are rendered -as images (``sphinx.ext.imgmath`` extension). If your docs build (``make -docs``) shows warnings like this:: +as images (``sphinx.ext.imgmath`` extension). + +Here is the extract from the :origin:`docs/conf.py` file, setting math renderer +to ``imgmath``: + +.. literalinclude:: ../conf.py + :language: python + :start-after: # sphinx.ext.imgmath setup + :end-before: # sphinx.ext.imgmath setup END + +If your docs build (``make docs``) shows warnings like this:: WARNING: dot(1) not found, for better output quality install \ graphviz from http://www.graphviz.org @@ -47,8 +75,6 @@ docs``) shows warnings like this:: you need to install additional packages on your build host, to get better HTML output. -.. _system requirements: - .. tabs:: .. group-tab:: Ubuntu / debian @@ -92,12 +118,38 @@ For PDF output you also need: $ sudo dnf install \ texlive-collection-fontsrecommended texlive-collection-latex \ - dejavu-sans-fonts dejavu-serif-fonts dejavu-sans-mono-fonts + dejavu-sans-fonts dejavu-serif-fonts dejavu-sans-mono-fonts \ + ImageMagick -.. _system requirements END: +.. _sh lint: -.. literalinclude:: ../conf.py - :language: python - :start-after: # sphinx.ext.imgmath setup - :end-before: # sphinx.ext.imgmath setup END +Lint shell scripts +================== + +.. _ShellCheck: https://github.com/koalaman/shellcheck + +To lint shell scripts, we use ShellCheck_ - A shell script static analysis tool. + +.. SNIP sh lint requirements + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code-block:: sh + + $ sudo apt install shellcheck + + .. group-tab:: Arch Linux + + .. code-block:: sh + + $ sudo pacman -S shellcheck + + .. group-tab:: Fedora / RHEL + + .. code-block:: sh + + $ sudo dnf install ShellCheck +.. SNAP sh lint requirements diff --git a/docs/admin/filtron.rst b/docs/admin/filtron.rst index 07dcb9bc5..93e430b1f 100644 --- a/docs/admin/filtron.rst +++ b/docs/admin/filtron.rst @@ -1,18 +1,51 @@ + +.. _searx filtron: + ========================== How to protect an instance ========================== -Searx depens on external search services. To avoid the abuse of these services +.. sidebar:: further reading + + - :ref:`filtron.sh` + - :ref:`nginx searx site` + + +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + +.. _filtron: https://github.com/asciimoo/filtron + +Searx depends on external search services. To avoid the abuse of these services it is advised to limit the number of requests processed by searx. -An application firewall, ``filtron`` solves exactly this problem. Information -on how to install it can be found at the `project page of filtron -<https://github.com/asciimoo/filtron>`__. +An application firewall, filtron_ solves exactly this problem. Filtron is just +a middleware between your web server (nginx, apache, ...) and searx, we describe +such infratructures in chapter: :ref:`architecture`. +filtron & go +============ + +.. _Go: https://golang.org/ +.. _filtron README: https://github.com/asciimoo/filtron/blob/master/README.md + +Filtron needs Go_ installed. If Go_ is preinstalled, filtron_ is simply +installed by ``go get`` package management (see `filtron README`_). If you use +filtron as middleware, a more isolated setup is recommended. To simplify such +an installation and the maintenance of, use our script :ref:`filtron.sh`. + +.. _Sample configuration of filtron: + Sample configuration of filtron =============================== +.. sidebar:: Tooling box + + - :origin:`/etc/filtron/rules.json <utils/templates/etc/filtron/rules.json>` + An example configuration can be find below. This configuration limits the access of: @@ -24,105 +57,104 @@ of: .. code:: json - [{ - "name":"search request", - "filters":[ - "Param:q", - "Path=^(/|/search)$" - ], - "interval":"<time-interval-in-sec (int)>", - "limit":"<max-request-number-in-interval (int)>", - "subrules":[ - { - "name":"roboagent limit", - "interval":"<time-interval-in-sec (int)>", - "limit":"<max-request-number-in-interval (int)>", - "filters":[ - "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)" - ], - "actions":[ - { - "name":"block", - "params":{ - "message":"Rate limit exceeded" - } - } - ] - }, - { - "name":"botlimit", - "limit":0, - "stop":true, - "filters":[ - "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)" - ], - "actions":[ - { - "name":"block", - "params":{ - "message":"Rate limit exceeded" - } - } - ] - }, - { - "name":"IP limit", - "interval":"<time-interval-in-sec (int)>", - "limit":"<max-request-number-in-interval (int)>", - "stop":true, - "aggregations":[ - "Header:X-Forwarded-For" - ], - "actions":[ - { - "name":"block", - "params":{ - "message":"Rate limit exceeded" - } - } - ] - }, - { - "name":"rss/json limit", - "interval":"<time-interval-in-sec (int)>", - "limit":"<max-request-number-in-interval (int)>", - "stop":true, - "filters":[ - "Param:format=(csv|json|rss)" - ], - "actions":[ - { - "name":"block", - "params":{ - "message":"Rate limit exceeded" - } - } - ] - }, - { - "name":"useragent limit", - "interval":"<time-interval-in-sec (int)>", - "limit":"<max-request-number-in-interval (int)>", - "aggregations":[ - "Header:User-Agent" + [ + { + "name": "search request", + "filters": [ + "Param:q", + "Path=^(/|/search)$" ], - "actions":[ - { - "name":"block", - "params":{ - "message":"Rate limit exceeded" - } - } + "interval": "<time-interval-in-sec (int)>" + "limit": "<max-request-number-in-interval (int)>", + "subrules": [ + { + "name": "missing Accept-Language", + "filters": ["!Header:Accept-Language"], + "limit": "<max-request-number-in-interval (int)>", + "stop": true, + "actions": [ + {"name":"log"}, + {"name": "block", + "params": {"message": "Rate limit exceeded"}} + ] + }, + { + "name": "suspiciously Connection=close header", + "filters": ["Header:Connection=close"], + "limit": "<max-request-number-in-interval (int)>", + "stop": true, + "actions": [ + {"name":"log"}, + {"name": "block", + "params": {"message": "Rate limit exceeded"}} + ] + }, + { + "name": "IP limit", + "interval": "<time-interval-in-sec (int)>" + "limit": "<max-request-number-in-interval (int)>", + "stop": true, + "aggregations": [ + "Header:X-Forwarded-For" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "rss/json limit", + "filters": [ + "Param:format=(csv|json|rss)" + ], + "interval": "<time-interval-in-sec (int)>" + "limit": "<max-request-number-in-interval (int)>", + "stop": true, + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "useragent limit", + "interval": "<time-interval-in-sec (int)>" + "limit": "<max-request-number-in-interval (int)>", + "aggregations": [ + "Header:User-Agent" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + } ] - } - ] - }] + } + ] +.. _filtron route request: Route request through filtron ============================= +.. sidebar:: further reading + + - :ref:`filtron.sh overview` + - :ref:`installation nginx` + - :ref:`installation apache` + Filtron can be started using the following command: .. code:: sh @@ -136,13 +168,24 @@ Use it along with ``nginx`` with the following example configuration. .. code:: nginx - location / { - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Scheme $scheme; - proxy_pass http://127.0.0.1:4004/; + # https://example.org/searx + + location /searx { + proxy_pass http://127.0.0.1:4004/; + + proxy_set_header Host $http_host; + proxy_set_header Connection $http_connection; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Script-Name /searx; } + location /searx/static { + /usr/local/searx/searx-src/searx/static; + } + + Requests are coming from port 4004 going through filtron and then forwarded to -port 8888 where a searx is being run. +port 8888 where a searx is being run. For a complete setup see: :ref:`nginx +searx site`. diff --git a/docs/admin/index.rst b/docs/admin/index.rst index b3c7f5119..c708c4ffa 100644 --- a/docs/admin/index.rst +++ b/docs/admin/index.rst @@ -3,9 +3,16 @@ Administrator documentation =========================== .. toctree:: - :maxdepth: 1 + :maxdepth: 2 + :caption: Contents installation + installation-searx + installation-uwsgi + installation-nginx + installation-apache + installation-docker + update-searx settings api architecture diff --git a/docs/admin/installation-apache.rst b/docs/admin/installation-apache.rst new file mode 100644 index 000000000..217e57718 --- /dev/null +++ b/docs/admin/installation-apache.rst @@ -0,0 +1,514 @@ +.. _installation apache: + +=================== +Install with apache +=================== + +.. _Apache: https://httpd.apache.org/ +.. _Apache Debian: + https://cwiki.apache.org/confluence/display/HTTPD/DistrosDefaultLayout#DistrosDefaultLayout-Debian,Ubuntu(Apachehttpd2.x): +.. _README.Debian: + https://salsa.debian.org/apache-team/apache2/raw/master/debian/apache2.README.Debian +.. _Apache Arch Linux: + https://wiki.archlinux.org/index.php/Apache_HTTP_Server +.. _Apache Fedora: + https://docs.fedoraproject.org/en-US/quick-docs/getting-started-with-apache-http-server/index.html +.. _Apache directives: + https://httpd.apache.org/docs/trunk/mod/directives.html +.. _Getting Started: + https://httpd.apache.org/docs/current/en/getting-started.html +.. _Terms Used to Describe Directives: + https://httpd.apache.org/docs/current/en/mod/directive-dict.html +.. _Configuration Files: + https://httpd.apache.org/docs/current/en/configuring.html +.. _ProxyPreserveHost: https://httpd.apache.org/docs/trunk/mod/mod_proxy.html#proxypreservehost +.. _LoadModule: + https://httpd.apache.org/docs/2.4/mod/mod_so.html#loadmodule +.. _DocumentRoot: + https://httpd.apache.org/docs/trunk/mod/core.html#documentroot +.. _Location: + https://httpd.apache.org/docs/trunk/mod/core.html#location +.. _uWSGI Apache support: + https://uwsgi-docs.readthedocs.io/en/latest/Apache.html +.. _mod_proxy_uwsgi: + https://uwsgi-docs.readthedocs.io/en/latest/Apache.html#mod-proxy-uwsgi + +.. sidebar:: further read + + - `Apache Arch Linux`_ + - `Apache Debian`_ and `README.Debian`_ + - `Apache Fedora`_ + - `Apache directives`_ + +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + +---- + +**Install** :ref:`apache searx site` using :ref:`filtron.sh <filtron.sh overview>` + +.. code:: bash + + $ sudo -H ./utils/filtron.sh apache install + +**Install** :ref:`apache searx site` using :ref:`morty.sh <morty.sh overview>` + +.. code:: bash + + $ sudo -H ./utils/morty.sh apache install + +---- + +The apache HTTP server +====================== + +If Apache_ is not installed, install it now. If apache_ is new to you, the +`Getting Started`_, `Configuration Files`_ and `Terms Used to Describe +Directives`_ documentation gives first orientation. There is also a list of +`Apache directives`_ *to keep in the pocket*. + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code:: sh + + sudo -H apt-get install apache2 + + .. group-tab:: Arch Linux + + .. code:: sh + + sudo -H pacman -S apache + sudo -H systemctl enable httpd + sudo -H systemctl start http + + .. group-tab:: Fedora / RHEL + + .. code:: sh + + sudo -H dnf install httpd + sudo -H systemctl enable httpd + sudo -H systemctl start httpd + +Now at http://localhost you should see any kind of *Welcome* or *Test* page. +How this default intro site is configured, depends on the linux distribution +(compare `Apache directives`_). + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code:: sh + + less /etc/apache2/sites-enabled/000-default.conf + + In this file, there is a line setting the `DocumentRoot`_ directive: + + .. code:: apache + + DocumentRoot /var/www/html + + And the *welcome* page is the HTML file at ``/var/www/html/index.html``. + + .. group-tab:: Arch Linux + + .. code:: sh + + less /etc/httpd/conf/httpd.conf + + In this file, there is a line setting the `DocumentRoot`_ directive: + + .. code:: apache + + DocumentRoot "/srv/http" + <Directory "/srv/http"> + Options Indexes FollowSymLinks + AllowOverride None + Require all granted + </Directory> + + The *welcome* page of Arch Linux is a page showing directory located at + ``DocumentRoot``. This is *directory* page is generated by the Module + `mod_autoindex <https://httpd.apache.org/docs/2.4/mod/mod_autoindex.html>`_: + + .. code:: apache + + LoadModule autoindex_module modules/mod_autoindex.so + ... + Include conf/extra/httpd-autoindex.conf + + .. group-tab:: Fedora / RHEL + + .. code:: sh + + less /etc/httpd/conf/httpd.conf + + In this file, there is a line setting the ``DocumentRoot`` directive: + + .. code:: apache + + DocumentRoot "/var/www/html" + ... + <Directory "/var/www"> + AllowOverride None + # Allow open access: + Require all granted + </Directory> + + On fresh installations, the ``/var/www`` is empty and the *default + welcome page* is shown, the configuration is located at:: + + less /etc/httpd/conf.d/welcome.conf + +.. _apache searx site: + +Apache Reverse Proxy +==================== + +.. sidebar:: public to the internet? + + If your searx instance is public, stop here and first install :ref:`filtron + reverse proxy <filtron.sh>` and :ref:`result proxy morty <morty.sh>`, see + :ref:`installation scripts`. If already done, follow setup: *searx via + filtron plus morty*. + +To setup a Apache revers proxy you have to enable the *headers* and *proxy* +modules and create a `Location`_ configuration for the searx site. In most +distributions you have to un-comment the lines in the main configuration file, +except in :ref:`The Debian Layout`. + +To pass the HTTP HOST header +With ProxyPreserveHost_ the incoming Host HTTP request header is passed to the +proxied host. + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + In the Apache setup, enable headers and proxy modules: + + .. code:: sh + + sudo -H a2enmod headers + sudo -H a2enmod proxy + sudo -H a2enmod proxy_http + + In :ref:`The Debian Layout` you create a ``searx.conf`` with the + ``<Location /searx >`` directive and save this file in the *sites + available* folder at ``/etc/apache2/sites-available``. To enable the + ``searx.conf`` use :man:`a2ensite`: + + .. code:: sh + + sudo -H a2ensite searx.conf + + .. group-tab:: Arch Linux + + In the ``/etc/httpd/conf/httpd.conf`` file, activate headers and proxy + modules (LoadModule_): + + .. code:: apache + + FIXME needs test + + LoadModule headers_module modules/mod_headers.so + LoadModule proxy_module modules/mod_proxy.so + LoadModule proxy_http_module modules/mod_proxy_http.so + + .. group-tab:: Fedora / RHEL + + In the ``/etc/httpd/conf/httpd.conf`` file, activate headers and proxy + modules (LoadModule_): + + .. code:: apache + + FIXME needs test + + LoadModule headers_module modules/mod_headers.so + LoadModule proxy_module modules/mod_proxy.so + LoadModule proxy_http_module modules/mod_proxy_http.so + +.. tabs:: + + .. group-tab:: searx via filtron plus morty + + Use this setup, if your instance is public to the internet, compare + figure: :ref:`architecture <arch public>` and :ref:`installation scripts`. + + 1. Configure a reverse proxy for :ref:`filtron <filtron.sh>`, listening on + *localhost 4004* (:ref:`filtron route request`): + + .. code:: apache + + <Location /searx > + + # SetEnvIf Request_URI "/searx" dontlog + # CustomLog /dev/null combined env=dontlog + + Require all granted + + Order deny,allow + Deny from all + #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + ProxyPreserveHost On + ProxyPass http://127.0.0.1:4004 + RequestHeader set X-Script-Name /searx + + </Location> + + 2. Configure reverse proxy for :ref:`morty <searx morty>`, listening on + *localhost 3000* + + .. code:: apache + + ProxyPreserveHost On + + <Location /morty > + + # SetEnvIf Request_URI "/morty" dontlog + # CustomLog /dev/null combined env=dontlog + + Require all granted + + Order deny,allow + Deny from all + #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + ProxyPass http://127.0.0.1:3000 + RequestHeader set X-Script-Name /morty + + </Location> + + Note that reverse proxy advised to be used in case of single-user or + low-traffic instances. For a fully result proxification add :ref:`morty's + <searx morty>` **public URL** to your :origin:`searx/settings.yml`: + + .. code:: yaml + + result_proxy: + # replace example.org with your server's public name + url : https://example.org/morty + + server: + image_proxy : True + +uWSGI support +============= + +Be warned, with this setup, your instance isn't :ref:`protected <searx +filtron>`, nevertheless it is good enough for intranet usage. In modern Linux +distributions, the `mod_proxy_uwsgi`_ is compiled into the *normal* apache +package and you need to install only the :ref:`uWSGI <searx uwsgi>` package: + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code:: sh + + sudo -H apt-get install uwsgi + + # Ubuntu =< 18.04 + sudo -H apt-get install libapache2-mod-proxy-uwsgi + + .. group-tab:: Arch Linux + + .. code:: sh + + sudo -H pacman -S uwsgi + + .. group-tab:: Fedora / RHEL + + .. code:: sh + + sudo -H dnf install uwsgi + +The next example shows a configuration using the `uWSGI Apache support`_ via +unix sockets and `mod_proxy_uwsgi`_. + +For socket communication, you have to activate ``socket = +/run/uwsgi/app/searx/socket`` and comment out the ``http = 127.0.0.1:8888`` +configuration in your :ref:`uwsgi ini file <uwsgi configuration>`. If not +already exists, create a folder for the unix sockets, which can be used by the +searx account (see :ref:`create searx user`): + +.. code:: bash + + sudo -H mkdir -p /run/uwsgi/app/searx/ + sudo -H chown -R searx:searx /run/uwsgi/app/searx/ + +If the server is public; to limit access to your intranet replace ``Allow from +all`` directive and replace ``192.168.0.0/16`` with your subnet IP/class. + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code:: apache + + LoadModule headers_module /usr/lib/apache2/mod_headers.so + LoadModule proxy_module /usr/lib/apache2/modules/mod_proxy.so + LoadModule proxy_uwsgi_module /usr/lib/apache2/modules/mod_proxy_uwsgi.so + + # SetEnvIf Request_URI /searx dontlog + # CustomLog /dev/null combined env=dontlog + + <Location /searx> + + Require all granted + Order deny,allow + Deny from all + # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + ProxyPreserveHost On + ProxyPass unix:/run/uwsgi/app/searx/socket|uwsgi://uwsgi-uds-searx/ + + </Location> + + .. group-tab:: Arch Linux + + .. code:: apache + + FIXME needs test + + LoadModule proxy_module modules/mod_proxy.so + LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so + + # SetEnvIf Request_URI /searx dontlog + # CustomLog /dev/null combined env=dontlog + + <Location /searx> + + Require all granted + Order deny,allow + Deny from all + # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + ProxyPreserveHost On + ProxyPass unix:/run/uwsgi/app/searx/socket|uwsgi://uwsgi-uds-searx/ + + </Location> + + .. group-tab:: Fedora / RHEL + + .. code:: apache + + FIXME needs test + + LoadModule proxy_module modules/mod_proxy.so + LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so + <IfModule proxy_uwsgi_module> + + # SetEnvIf Request_URI /searx dontlog + # CustomLog /dev/null combined env=dontlog + + <Location /searx> + + Require all granted + Order deny,allow + Deny from all + # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + ProxyPreserveHost On + ProxyPass unix:/run/uwsgi/app/searx/socket|uwsgi://uwsgi-uds-searx/ + + </Location> + + </IfModule> + + .. group-tab:: old mod_wsgi + + We show this only for historical reasons, DON'T USE `mod_uwsgi + <https://uwsgi-docs.readthedocs.io/en/latest/Apache.html#mod-uwsgi>`_. + ANYMORE! + + .. code:: apache + + <IfModule mod_uwsgi.c> + + # SetEnvIf Request_URI "/searx" dontlog + # CustomLog /dev/null combined env=dontlog + + <Location /searx > + + Require all granted + + Options FollowSymLinks Indexes + SetHandler uwsgi-handler + uWSGISocket /run/uwsgi/app/searx/socket + + Order deny,allow + Deny from all + # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + </Location> + + </IfModule> + +.. _restart apache: + +Restart service +=============== + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code:: sh + + sudo -H systemctl restart apache2 + sudo -H service uwsgi restart searx + + .. group-tab:: Arch Linux + + .. code:: sh + + sudo -H systemctl restart httpd + sudo -H systemctl restart uwsgi@searx + + .. group-tab:: Fedora / RHEL + + .. code:: sh + + sudo -H systemctl restart httpd + sudo -H touch /etc/uwsgi.d/searx.ini + + +disable logs +============ + +For better privacy you can disable Apache logs. In the examples above activate +one of the lines and `restart apache`_:: + + + # SetEnvIf Request_URI "/searx" dontlog + # CustomLog /dev/null combined env=dontlog + +The ``CustomLog`` directive disable logs for the whole (virtual) server, use it +when the URL of the service does not have a path component (``/searx``) / is +located at root (``/``). + +.. _The Debian Layout: + +The Debian Layout +================= + +Be aware that the Debian layout is quite different from the standard Apache +configuration. For details look at the README.Debian_ +(``/usr/share/doc/apache2/README.Debian.gz``). Some commands you should know on +Debian: + +* :man:`apache2ctl`: Apache HTTP server control interface +* :man:`a2enmod`, :man:`a2dismod`: switch on/off modules +* :man:`a2enconf`, :man:`a2disconf`: switch on/off configurations +* :man:`a2ensite`, :man:`a2dissite`: switch on/off sites diff --git a/docs/admin/installation-docker.rst b/docs/admin/installation-docker.rst new file mode 100644 index 000000000..340e66319 --- /dev/null +++ b/docs/admin/installation-docker.rst @@ -0,0 +1,28 @@ +.. _installation docker: + +=================== +Docker installation +=================== + +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + +Make sure you have installed Docker. For instance, you can deploy searx like this: + +.. code:: sh + + docker pull wonderfall/searx + docker run -d --name searx -p $PORT:8888 wonderfall/searx + +Go to ``http://localhost:$PORT``. + +See https://hub.docker.com/r/wonderfall/searx/ for more informations. It's also +possible to build searx from the embedded Dockerfile. + +.. code:: sh + + git clone https://github.com/asciimoo/searx.git + cd searx + docker build -t whatever/searx . diff --git a/docs/admin/installation-nginx.rst b/docs/admin/installation-nginx.rst new file mode 100644 index 000000000..5e32d9684 --- /dev/null +++ b/docs/admin/installation-nginx.rst @@ -0,0 +1,381 @@ +.. _installation nginx: + +================== +Install with nginx +================== + +.. _nginx: + https://docs.nginx.com/nginx/admin-guide/ +.. _nginx server configuration: + https://docs.nginx.com/nginx/admin-guide/web-server/web-server/#setting-up-virtual-servers +.. _nginx beginners guide: + http://nginx.org/en/docs/beginners_guide.html +.. _Getting Started wiki: + https://www.nginx.com/resources/wiki/start/ +.. _uWSGI support from nginx: + https://uwsgi-docs.readthedocs.io/en/latest/Nginx.html +.. _uwsgi_params: + https://uwsgi-docs.readthedocs.io/en/latest/Nginx.html#configuring-nginx +.. _SCRIPT_NAME: + https://werkzeug.palletsprojects.com/en/1.0.x/wsgi/#werkzeug.wsgi.get_script_name + +.. sidebar:: further reading + + - nginx_ + - `nginx beginners guide`_ + - `nginx server configuration`_ + - `Getting Started wiki`_ + - `uWSGI support from nginx`_ + +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + +---- + +**Install** :ref:`nginx searx site` using :ref:`filtron.sh <filtron.sh overview>` + +.. code:: bash + + $ sudo -H ./utils/filtron.sh nginx install + +**Install** :ref:`nginx searx site` using :ref:`morty.sh <morty.sh overview>` + +.. code:: bash + + $ sudo -H ./utils/morty.sh nginx install + +---- + + +The nginx HTTP server +===================== + +If nginx_ is not installed (uwsgi will not work with the package nginx-light), +install it now. + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code:: sh + + sudo -H apt-get install nginx + + .. group-tab:: Arch Linux + + .. code-block:: sh + + sudo -H pacman -S nginx-mainline + sudo -H systemctl enable nginx + sudo -H systemctl start nginx + + .. group-tab:: Fedora / RHEL + + .. code-block:: sh + + sudo -H dnf install nginx + sudo -H systemctl enable nginx + sudo -H systemctl start nginx + +Now at http://localhost you should see a *Welcome to nginx!* page, on Fedora you +see a *Fedora Webserver - Test Page*. The test page comes from the default +`nginx server configuration`_. How this default intro site is configured, +depends on the linux distribution: + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code:: sh + + less /etc/nginx/nginx.conf + + there is a line including site configurations from: + + .. code:: nginx + + include /etc/nginx/sites-enabled/*; + + .. group-tab:: Arch Linux + + .. code-block:: sh + + less /etc/nginx/nginx.conf + + in there is a configuration section named ``server``: + + .. code-block:: nginx + + server { + listen 80; + server_name localhost; + # ... + } + + .. group-tab:: Fedora / RHEL + + .. code-block:: sh + + less /etc/nginx/nginx.conf + + there is a line including site configurations from: + + .. code:: nginx + + include /etc/nginx/conf.d/*.conf; + +.. _nginx searx site: + +A nginx searx site +================== + +.. sidebar:: public to the internet? + + If your searx instance is public, stop here and first install :ref:`filtron + reverse proxy <filtron.sh>` and :ref:`result proxy morty <morty.sh>`, see + :ref:`installation scripts`. If already done, follow setup: *searx via + filtron plus morty*. + +Now you have to create a configuration for the searx site. If nginx_ is new to +you, the `nginx beginners guide`_ is a good starting point and the `Getting +Started wiki`_ is always a good resource *to keep in the pocket*. + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + Create configuration at ``/etc/nginx/sites-available/searx`` and place a + symlink to sites-enabled: + + .. code:: sh + + sudo -H ln -s /etc/nginx/sites-available/searx /etc/nginx/sites-enabled/searx + + .. group-tab:: Arch Linux + + In the ``/etc/nginx/nginx.conf`` file, replace the configuration section + named ``server``. + + .. group-tab:: Fedora / RHEL + + Create configuration at ``/etc/nginx/conf.d/searx`` and place a + symlink to sites-enabled: + +.. tabs:: + + .. group-tab:: searx via filtron plus morty + + Use this setup, if your instance is public to the internet, compare + figure: :ref:`architecture <arch public>` and :ref:`installation scripts`. + + 1. Configure a reverse proxy for :ref:`filtron <filtron.sh>`, listening on + *localhost 4004* (:ref:`filtron route request`): + + .. code:: nginx + + # https://example.org/searx + + location /searx { + proxy_pass http://127.0.0.1:4004/; + + proxy_set_header Host $http_host; + proxy_set_header Connection $http_connection; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Script-Name /searx; + } + + location /searx/static { + /usr/local/searx/searx-src/searx/static; + } + + + 2. Configure reverse proxy for :ref:`morty <searx morty>`, listening on + *localhost 3000*: + + .. code:: nginx + + # https://example.org/morty + + location /morty { + proxy_pass http://127.0.0.1:3000/; + + proxy_set_header Host $http_host; + proxy_set_header Connection $http_connection; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Scheme $scheme; + } + + Note that reverse proxy advised to be used in case of single-user or + low-traffic instances. For a fully result proxification add :ref:`morty's + <searx morty>` **public URL** to your :origin:`searx/settings.yml`: + + .. code:: yaml + + result_proxy: + # replace example.org with your server's public name + url : https://example.org/morty + + server: + image_proxy : True + + + .. group-tab:: proxy or uWSGI + + Be warned, with this setup, your instance isn't :ref:`protected <searx + filtron>`. Nevertheless it is good enough for intranet usage and it is a + excellent example of; *how different services can be set up*. The next + example shows a reverse proxy configuration wrapping the :ref:`searx-uWSGI + application <uwsgi configuration>`, listening on ``http = + 127.0.0.1:8888``. + + .. code:: nginx + + # https://hostname.local/ + + location / { + proxy_pass http://127.0.0.1:8888; + + proxy_set_header Host $host; + proxy_set_header Connection $http_connection; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Scheme $scheme; + proxy_buffering off; + } + + Alternatively you can use the `uWSGI support from nginx`_ via unix + sockets. For socket communication, you have to activate ``socket = + /run/uwsgi/app/searx/socket`` and comment out the ``http = + 127.0.0.1:8888`` configuration in your :ref:`uwsgi ini file <uwsgi + configuration>`. + + The example shows a nginx virtual ``server`` configuration, listening on + port 80 (IPv4 and IPv6 http://[::]:80). The uWSGI app is configured at + location ``/`` by importing the `uwsgi_params`_ and passing requests to + the uWSGI socket (``uwsgi_pass``). The ``server``\'s root points to the + :ref:`searx-src clone <searx-src>` and wraps directly the + :origin:`searx/static/` content at ``location /static``. + + .. code:: nginx + + server { + # replace hostname.local with your server's name + server_name hostname.local; + + listen 80; + listen [::]:80; + + location / { + include uwsgi_params; + uwsgi_pass unix:/run/uwsgi/app/searx/socket; + } + + root /usr/local/searx/searx-src/searx; + location /static { } + } + + If not already exists, create a folder for the unix sockets, which can be + used by the searx account: + + .. code:: bash + + mkdir -p /run/uwsgi/app/searx/ + sudo -H chown -R searx:searx /run/uwsgi/app/searx/ + + .. group-tab:: \.\. at subdir URL + + Be warned, with these setups, your instance isn't :ref:`protected <searx + filtron>`. The examples are just here to demonstrate how to export the + searx application from a subdirectory URL ``https://example.org/searx/``. + + .. code:: nginx + + # https://hostname.local/searx + + location /searx { + proxy_pass http://127.0.0.1:8888; + + proxy_set_header Host $host; + proxy_set_header Connection $http_connection; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Script-Name /searx; + proxy_buffering off; + } + + location /searx/static { + alias /usr/local/searx/searx-src/searx/static; + } + + The ``X-Script-Name /searx`` is needed by the searx implementation to + calculate relative URLs correct. The next example shows a uWSGI + configuration. Since there are no HTTP headers in a (u)WSGI protocol, the + value is shipped via the SCRIPT_NAME_ in the WSGI environment. + + .. code:: nginx + + # https://hostname.local/searx + + location /searx { + uwsgi_param SCRIPT_NAME /searx; + include uwsgi_params; + uwsgi_pass unix:/run/uwsgi/app/searx/socket; + } + + location /searx/static { + alias /usr/local/searx/searx-src/searx; + } + + For searx to work correctly the ``base_url`` must be set in the + :origin:`searx/settings.yml`. + + .. code:: yaml + + server: + # replace example.org with your server's public name + base_url : https://example.org/searx/ + + +Restart service: + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code:: sh + + sudo -H systemctl restart nginx + sudo -H service uwsgi restart searx + + .. group-tab:: Arch Linux + + .. code:: sh + + sudo -H systemctl restart nginx + sudo -H systemctl restart uwsgi@searx + + .. group-tab:: Fedora + + .. code:: sh + + sudo -H systemctl restart nginx + sudo -H touch /etc/uwsgi.d/searx.ini + + +Disable logs +============ + +For better privacy you can disable nginx logs in ``/etc/nginx/nginx.conf``. + +.. code:: nginx + + http { + # ... + access_log /dev/null; + error_log /dev/null; + # ... + } diff --git a/docs/admin/installation-searx.rst b/docs/admin/installation-searx.rst new file mode 100644 index 000000000..f1d486021 --- /dev/null +++ b/docs/admin/installation-searx.rst @@ -0,0 +1,92 @@ +.. _installation basic: + +========================= +Step by step installation +========================= + +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + +Step by step installation with virtualenv. For Ubuntu, be sure to have enable +universe repository. + +.. _install packages: + +Install packages +================ + +.. kernel-include:: $DOCS_BUILD/includes/searx.rst + :start-after: START distro-packages + :end-before: END distro-packages + +.. hint:: + + This installs also the packages needed by :ref:`searx uwsgi` + +.. _create searx user: + +Create user +=========== + +.. kernel-include:: $DOCS_BUILD/includes/searx.rst + :start-after: START create user + :end-before: END create user + +.. _searx-src: + +install searx & dependencies +============================ + +Start a interactive shell from new created user and clone searx: + +.. kernel-include:: $DOCS_BUILD/includes/searx.rst + :start-after: START clone searx + :end-before: END clone searx + +In the same shell create *virtualenv*: + +.. kernel-include:: $DOCS_BUILD/includes/searx.rst + :start-after: START create virtualenv + :end-before: END create virtualenv + +To install searx's dependencies, exit the searx *bash* session you opened above +and restart a new. Before install, first check if your *virualenv* was sourced +from the login (*~/.profile*): + +.. kernel-include:: $DOCS_BUILD/includes/searx.rst + :start-after: START manage.sh update_packages + :end-before: END manage.sh update_packages + +.. tip:: + + Open a second terminal for the configuration tasks and left the ``(searx)$`` + terminal open for the tasks below. + +Configuration +============== + +Create a copy of the :origin:`searx/settings.yml` configuration file in system's +*/etc* folder. Configure like shown below -- replace ``searx@\$(uname -n)`` with +a name of your choice -- *and/or* edit ``/etc/searx/settings.yml`` if necessary. + +.. kernel-include:: $DOCS_BUILD/includes/searx.rst + :start-after: START searx config + :end-before: END searx config + +Check +===== + +To check your searx setup, optional enable debugging and start the *webapp*. +Searx looks at the exported environment ``$SEARX_SETTINGS_PATH`` for a +configuration file. + +.. kernel-include:: $DOCS_BUILD/includes/searx.rst + :start-after: START check searx installation + :end-before: END check searx installation + +If everything works fine, hit ``[CTRL-C]`` to stop the *webapp* and disable the +debug option in ``settings.yml``. You can now exit searx user bash (enter exit +command twice). At this point searx is not demonized; uwsgi allows this. + diff --git a/docs/admin/installation-uwsgi.rst b/docs/admin/installation-uwsgi.rst new file mode 100644 index 000000000..ac4c463b9 --- /dev/null +++ b/docs/admin/installation-uwsgi.rst @@ -0,0 +1,149 @@ +.. _searx uwsgi: + +===== +uwsgi +===== + +.. sidebar:: further reading + + - `systemd.unit`_ + - `uWSGI Emperor`_ + +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + + +.. _systemd.unit: https://www.freedesktop.org/software/systemd/man/systemd.unit.html +.. _One service per app in systemd: + https://uwsgi-docs.readthedocs.io/en/latest/Systemd.html#one-service-per-app-in-systemd +.. _uWSGI Emperor: + https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html +.. _uwsgi ini file: + https://uwsgi-docs.readthedocs.io/en/latest/Configuration.html#ini-files +.. _systemd unit template: + http://0pointer.de/blog/projects/instances.html + + +Origin uWSGI +============ + +How uWSGI is implemented by distributors is different. uWSGI itself +recommend two methods + +`systemd.unit`_ template files as described here `One service per app in systemd`_. + + There is one `systemd unit template`_ and one `uwsgi ini file`_ per uWSGI-app + placed at dedicated locations. Take archlinux and a searx.ini as example:: + + unit template --> /usr/lib/systemd/system/uwsgi@.service + uwsgi ini files --> /etc/uwsgi/searx.ini + + The searx app can be maintained as know from common systemd units:: + + systemctl enable uwsgi@searx + systemctl start uwsgi@searx + systemctl restart uwsgi@searx + systemctl stop uwsgi@searx + +The `uWSGI Emperor`_ mode which fits for maintaining a large range of uwsgi apps. + + The Emperor mode is a special uWSGI instance that will monitor specific + events. The Emperor mode (service) is started by a (common, not template) + systemd unit. The Emperor service will scan specific directories for `uwsgi + ini file`_\s (also know as *vassals*). If a *vassal* is added, removed or the + timestamp is modified, a corresponding action takes place: a new uWSGI + instance is started, reload or stopped. Take Fedora and a searx.ini as + example:: + + to start a new searx instance create --> /etc/uwsgi.d/searx.ini + to reload the instance edit timestamp --> touch /etc/uwsgi.d/searx.ini + to stop instance remove ini --> rm /etc/uwsgi.d/searx.ini + +Distributors +============ + +The `uWSGI Emperor`_ mode and `systemd unit template`_ is what the distributors +mostly offer their users, even if they differ in the way they implement both +modes and their defaults. Another point they might differ is the packaging of +plugins (if so, compare :ref:`install packages`) and what the default python +interpreter is (python2 vs. python3). + +Fedora starts a Emperor by default, while archlinux does not start any uwsgi +service by default. Worth to know; debian (ubuntu) follow a complete different +approach. *debian*: your are familiar with the apache infrastructure? .. they +do similar for the uWSGI infrastructure (with less comfort), the folders are:: + + /etc/uwsgi/apps-available/ + /etc/uwsgi/apps-enabled/ + +The `uwsgi ini file`_ is enabled by a symbolic link:: + + ln -s /etc/uwsgi/apps-available/searx.ini /etc/uwsgi/apps-enabled/ + +From debian's documentation (``/usr/share/doc/uwsgi/README.Debian.gz``): You +could control specific instance(s) by issuing:: + + service uwsgi <command> <confname> <confname> ... + + sudo -H service uwsgi start searx + sudo -H service uwsgi stop searx + +My experience is, that this command is a bit buggy. + +.. _uwsgi configuration: + +Alltogether +=========== + +Create the configuration ini-file according to your distribution (see below) and +restart the uwsgi application. + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. kernel-include:: $DOCS_BUILD/includes/searx.rst + :start-after: START searx uwsgi-description ubuntu-20.04 + :end-before: END searx uwsgi-description ubuntu-20.04 + + + .. group-tab:: Arch Linux + + .. kernel-include:: $DOCS_BUILD/includes/searx.rst + :start-after: START searx uwsgi-description arch + :end-before: END searx uwsgi-description arch + + + .. group-tab:: Fedora / RHEL + + .. kernel-include:: $DOCS_BUILD/includes/searx.rst + :start-after: START searx uwsgi-description fedora + :end-before: END searx uwsgi-description fedora + + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. kernel-include:: $DOCS_BUILD/includes/searx.rst + :code: ini + :start-after: START searx uwsgi-appini ubuntu-20.04 + :end-before: END searx uwsgi-appini ubuntu-20.04 + + .. group-tab:: Arch Linux + + .. kernel-include:: $DOCS_BUILD/includes/searx.rst + :code: ini + :start-after: START searx uwsgi-appini arch + :end-before: END searx uwsgi-appini arch + + .. group-tab:: Fedora / RHEL + + .. kernel-include:: $DOCS_BUILD/includes/searx.rst + :code: ini + :start-after: START searx uwsgi-appini fedora + :end-before: END searx uwsgi-appini fedora + + diff --git a/docs/admin/installation.rst b/docs/admin/installation.rst index 15800fc01..167c300fe 100644 --- a/docs/admin/installation.rst +++ b/docs/admin/installation.rst @@ -4,346 +4,63 @@ Installation ============ -.. contents:: - :depth: 3 +*You're spoilt for choice*, choose your preferred method of installation. -Basic installation -================== +- :ref:`installation docker` +- :ref:`installation scripts` +- :ref:`installation basic` -Step by step installation for Debian/Ubuntu with virtualenv. For Ubuntu, be sure -to have enable universe repository. +The :ref:`installation basic` is good enough for intranet usage and it is a +excellent illustration of *how a searx instance is build up*. If you place your +instance public to the internet you should really consider to install a +:ref:`filtron reverse proxy <filtron.sh>` and for privacy a :ref:`result proxy +<morty.sh>` is mandatory. -Install packages: +Therefore, if you do not have any special preferences, its recommend to use the +:ref:`installation docker` or the `Installation scripts`_ from our :ref:`tooling +box <toolboxing>` as described below. -.. code:: sh +.. _installation scripts: - $ sudo -H apt-get install \ - git build-essential libxslt-dev \ - python-dev python-virtualenv python-babel \ - zlib1g-dev libffi-dev libssl-dev +Installation scripts +==================== -Install searx: +.. sidebar:: Update OS first! -.. code:: sh + To avoid unwanted side effects, update your OS before installing searx. - cd /usr/local - sudo -H git clone https://github.com/asciimoo/searx.git - sudo -H useradd searx -d /usr/local/searx - sudo -H chown searx:searx -R /usr/local/searx +The following will install a setup as shown in :ref:`architecture`. First you +need to get a clone. The clone is only needed for the installation procedure +and some maintenance tasks (alternatively you can create your own fork). -Install dependencies in a virtualenv: +.. code:: bash -.. code:: sh + $ cd ~/Downloads + $ git clone https://github.com/asciimoo/searx searx + $ cd searx - cd /usr/local/searx - sudo -H -u searx -i +**Install** :ref:`searx service <searx.sh>` -.. code:: sh +This installs searx as described in :ref:`installation basic`. - (searx)$ virtualenv searx-ve - (searx)$ . ./searx-ve/bin/activate - (searx)$ ./manage.sh update_packages +.. code:: bash -Configuration -============== + $ sudo -H ./utils/searx.sh install all -.. code:: sh +**Install** :ref:`filtron reverse proxy <filtron.sh>` - sed -i -e "s/ultrasecretkey/`openssl rand -hex 16`/g" searx/settings.yml +.. code:: bash -Edit searx/settings.yml if necessary. + $ sudo -H ./utils/filtron.sh install all -Check -===== +**Install** :ref:`result proxy <morty.sh>` -Start searx: +.. code:: bash -.. code:: sh + $ sudo -H ./utils/morty.sh install all - python searx/webapp.py +If all services are running fine, you can add it to your HTTP server: -Go to http://localhost:8888 +- :ref:`installation apache` +- :ref:`installation nginx` -If everything works fine, disable the debug option in settings.yml: - -.. code:: sh - - sed -i -e "s/debug : True/debug : False/g" searx/settings.yml - -At this point searx is not demonized ; uwsgi allows this. - -You can exit the virtualenv and the searx user bash (enter exit command -twice). - -uwsgi -===== - -Install packages: - -.. code:: sh - - sudo -H apt-get install \ - uwsgi uwsgi-plugin-python - -Create the configuration file ``/etc/uwsgi/apps-available/searx.ini`` with this -content: - -.. code:: ini - - [uwsgi] - # Who will run the code - uid = searx - gid = searx - - # disable logging for privacy - disable-logging = true - - # Number of workers (usually CPU count) - workers = 4 - - # The right granted on the created socket - chmod-socket = 666 - - # Plugin to use and interpretor config - single-interpreter = true - master = true - plugin = python - lazy-apps = true - enable-threads = true - - # Module to import - module = searx.webapp - - # Support running the module from a webserver subdirectory. - route-run = fixpathinfo: - - # Virtualenv and python path - virtualenv = /usr/local/searx/searx-ve/ - pythonpath = /usr/local/searx/ - chdir = /usr/local/searx/searx/ - -Activate the uwsgi application and restart: - -.. code:: sh - - cd /etc/uwsgi/apps-enabled - ln -s ../apps-available/searx.ini - /etc/init.d/uwsgi restart - -Web server -========== - -with nginx ----------- - -If nginx is not installed (uwsgi will not work with the package -nginx-light): - -.. code:: sh - - sudo -H apt-get install nginx - -Hosted at / -~~~~~~~~~~~ - -Create the configuration file ``/etc/nginx/sites-available/searx`` with this -content: - -.. code:: nginx - - server { - listen 80; - server_name searx.example.com; - root /usr/local/searx/searx; - - location /static { - } - - location / { - include uwsgi_params; - uwsgi_pass unix:/run/uwsgi/app/searx/socket; - } - } - -Create a symlink to sites-enabled: - -.. code:: sh - - sudo -H ln -s /etc/nginx/sites-available/searx /etc/nginx/sites-enabled/searx - -Restart service: - -.. code:: sh - - sudo -H service nginx restart - sudo -H service uwsgi restart - -from subdirectory URL (/searx) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Add this configuration in the server config file -``/etc/nginx/sites-enabled/default``: - -.. code:: nginx - - location /searx/static { - alias /usr/local/searx/searx/static; - } - - location /searx { - uwsgi_param SCRIPT_NAME /searx; - include uwsgi_params; - uwsgi_pass unix:/run/uwsgi/app/searx/socket; - } - - -**OR** using reverse proxy (Please, note that reverse proxy advised to be used -in case of single-user or low-traffic instances.) - -.. code:: nginx - - location /searx/static { - alias /usr/local/searx/searx/static; - } - - location /searx { - proxy_pass http://127.0.0.1:8888; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Scheme $scheme; - proxy_set_header X-Script-Name /searx; - proxy_buffering off; - } - - -Enable ``base_url`` in ``searx/settings.yml`` - -.. code:: yaml - - base_url : http://your.domain.tld/searx/ - -Restart service: - -.. code:: sh - - sudo -H service nginx restart - sudo -H service uwsgi restart - -disable logs -^^^^^^^^^^^^ - -for better privacy you can disable nginx logs about searx. - -how to proceed: below ``uwsgi_pass`` in ``/etc/nginx/sites-available/default`` -add: - -.. code:: nginx - - access_log /dev/null; - error_log /dev/null; - -Restart service: - -.. code:: sh - - sudo -H service nginx restart - -with apache ------------ - -Add wsgi mod: - -.. code:: sh - - sudo -H apt-get install libapache2-mod-uwsgi - sudo -H a2enmod uwsgi - -Add this configuration in the file ``/etc/apache2/apache2.conf``: - -.. code:: apache - - <Location /> - Options FollowSymLinks Indexes - SetHandler uwsgi-handler - uWSGISocket /run/uwsgi/app/searx/socket - </Location> - -Note that if your instance of searx is not at the root, you should change -``<Location />`` by the location of your instance, like ``<Location /searx>``. - -Restart Apache: - -.. code:: sh - - sudo -H /etc/init.d/apache2 restart - -disable logs -~~~~~~~~~~~~ - -For better privacy you can disable Apache logs. - -.. warning:: - - You can only disable logs for the whole (virtual) server not for a specific - path. - -Go back to ``/etc/apache2/apache2.conf`` and above ``<Location />`` add: - -.. code:: apache - - CustomLog /dev/null combined - -Restart Apache: - -.. code:: sh - - sudo -H /etc/init.d/apache2 restart - -How to update -============= - -.. code:: sh - - cd /usr/local/searx - sudo -H -u searx -i - -.. code:: sh - - (searx)$ . ./searx-ve/bin/activate - (searx)$ git stash - (searx)$ git pull origin master - (searx)$ git stash apply - (searx)$ ./manage.sh update_packages - -.. code:: sh - - sudo -H service uwsgi restart - -Docker -====== - -Make sure you have installed Docker. For instance, you can deploy searx like this: - -.. code:: sh - - docker pull wonderfall/searx - docker run -d --name searx -p $PORT:8888 wonderfall/searx - -Go to ``http://localhost:$PORT``. - -See https://hub.docker.com/r/wonderfall/searx/ for more informations. It's also -possible to build searx from the embedded Dockerfile. - -.. code:: sh - - git clone https://github.com/asciimoo/searx.git - cd searx - docker build -t whatever/searx . - -References -========== - -* https://about.okhin.fr/posts/Searx/ with some additions - -* How to: `Setup searx in a couple of hours with a free SSL certificate - <https://www.reddit.com/r/privacytoolsIO/comments/366kvn/how_to_setup_your_own_privacy_respecting_search/>`__ diff --git a/docs/admin/morty.rst b/docs/admin/morty.rst index 7d7b34492..2858fde3c 100644 --- a/docs/admin/morty.rst +++ b/docs/admin/morty.rst @@ -1,7 +1,14 @@ + +.. _searx morty: + ========================= How to setup result proxy ========================= +.. sidebar:: further reading + + - :ref:`morty.sh` + .. _morty: https://github.com/asciimoo/morty .. _morty's README: https://github.com/asciimoo/morty diff --git a/docs/admin/settings.rst b/docs/admin/settings.rst index 2bfbae35c..8b1cb8dca 100644 --- a/docs/admin/settings.rst +++ b/docs/admin/settings.rst @@ -4,11 +4,17 @@ ``settings.yml`` ================ +This page describe the options possibilities of the :origin:`searx/settings.yml` +file. + .. sidebar:: Further reading .. - :ref:`search API` -This page describe the options possibilities of the settings.yml file. +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry .. _settings global: diff --git a/docs/admin/update-searx.rst b/docs/admin/update-searx.rst new file mode 100644 index 000000000..c74c9568a --- /dev/null +++ b/docs/admin/update-searx.rst @@ -0,0 +1,23 @@ +.. _update searx: + +============= +How to update +============= + +.. code:: sh + + sudo -H -u searx -i + (searx)$ git stash + (searx)$ git pull origin master + (searx)$ git stash apply + (searx)$ ./manage.sh update_packages + +Restart uwsgi: + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code:: sh + + sudo -H systemctl restart uwsgi diff --git a/docs/blog/index.rst b/docs/blog/index.rst index 94847ad62..5a8135fdc 100644 --- a/docs/blog/index.rst +++ b/docs/blog/index.rst @@ -3,7 +3,8 @@ Blog ==== .. toctree:: - :maxdepth: 1 + :maxdepth: 2 + :caption: Contents python3 admin diff --git a/docs/build-templates/filtron.rst b/docs/build-templates/filtron.rst new file mode 100644 index 000000000..83a451820 --- /dev/null +++ b/docs/build-templates/filtron.rst @@ -0,0 +1,52 @@ +.. START create user + +.. tabs:: + + .. group-tab:: bash + + .. code-block:: sh + + $ sudo -H useradd --shell /bin/bash --system \\ + --home-dir "$SERVICE_HOME" \\ + --comment "Privacy-respecting metasearch engine" $SERVICE_USER + + $ sudo -H mkdir "$SERVICE_HOME" + $ sudo -H chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME" + +.. END create user + +.. START install go + +.. tabs:: + + .. group-tab:: bash + + .. code-block:: bash + + $ cat > "$GO_ENV" <<EOF + export GOPATH=${SERVICE_HOME}/go-apps + export PATH=\$PATH:${SERVICE_HOME}/local/go/bin:\$GOPATH/bin + EOF + $ sudo -i -u "${SERVICE_USER}" + (${SERVICE_USER}) $ echo 'source $GO_ENV' >> ~/.profile + (${SERVICE_USER}) $ mkdir ${SERVICE_HOME}/local + (${SERVICE_USER}) $ wget --progress=bar -O "${GO_TAR}" \\ + "${GO_PKG_URL}" + (${SERVICE_USER}) $ tar -C ${SERVICE_HOME}/local/go -xzf "${GO_TAR}" + (${SERVICE_USER}) $ which go + ${SERVICE_HOME}/local/go/bin/go + +.. END install go + +.. START install filtron + +.. tabs:: + + .. group-tab:: bash + + .. code-block:: bash + + $ sudo -i -u "${SERVICE_USER}" + (${SERVICE_USER}) $ go get -v -u github.com/asciimoo/filtron + +.. END install filtron diff --git a/docs/build-templates/morty.rst b/docs/build-templates/morty.rst new file mode 100644 index 000000000..4a5d1f27f --- /dev/null +++ b/docs/build-templates/morty.rst @@ -0,0 +1,52 @@ +.. START create user + +.. tabs:: + + .. group-tab:: bash + + .. code-block:: sh + + $ sudo -H useradd --shell /bin/bash --system \\ + --home-dir "$SERVICE_HOME" \\ + --comment "Privacy-respecting metasearch engine" $SERVICE_USER + + $ sudo -H mkdir "$SERVICE_HOME" + $ sudo -H chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME" + +.. END create user + +.. START install go + +.. tabs:: + + .. group-tab:: bash + + .. code-block:: bash + + $ cat > "$GO_ENV" <<EOF + export GOPATH=${SERVICE_HOME}/go-apps + export PATH=\$PATH:${SERVICE_HOME}/local/go/bin:\$GOPATH/bin + EOF + $ sudo -i -u "${SERVICE_USER}" + (${SERVICE_USER}) $ echo 'source $GO_ENV' >> ~/.profile + (${SERVICE_USER}) $ mkdir ${SERVICE_HOME}/local + (${SERVICE_USER}) $ wget --progress=bar -O "${GO_TAR}" \\ + "${GO_PKG_URL}" + (${SERVICE_USER}) $ tar -C ${SERVICE_HOME}/local/go -xzf "${GO_TAR}" + (${SERVICE_USER}) $ which go + ${SERVICE_HOME}/local/go/bin/go + +.. END install go + +.. START install morty + +.. tabs:: + + .. group-tab:: bash + + .. code-block:: bash + + $ sudo -i -u "${SERVICE_USER}" + (${SERVICE_USER}) $ go get -v -u github.com/asciimoo/morty + +.. END install morty diff --git a/docs/build-templates/searx.rst b/docs/build-templates/searx.rst new file mode 100644 index 000000000..5cb70e95f --- /dev/null +++ b/docs/build-templates/searx.rst @@ -0,0 +1,192 @@ +.. template evaluated by: ./utils/searx.sh docs +.. hint: all dollar-names are variables, dollar sign itself is quoted by: \\$ + +.. START distro-packages + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code-block:: sh + + $ sudo -H apt-get install -y \\ +${debian} + + .. group-tab:: Arch Linux + + .. code-block:: sh + + $ sudo -H pacman -S --noconfirm \\ +${arch} + + .. group-tab:: Fedora / RHEL + + .. code-block:: sh + + $ sudo -H dnf install -y \\ +${fedora} + +.. END distro-packages + +.. START build-packages + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code-block:: sh + + $ sudo -H apt-get install -y \\ +${debian_build} + + .. group-tab:: Arch Linux + + .. code-block:: sh + + $ sudo -H pacman -S --noconfirm \\ +${arch_build} + + .. group-tab:: Fedora / RHEL + + .. code-block:: sh + + $ sudo -H dnf install -y \\ +${fedora_build} + +.. END build-packages + +.. START create user + +.. tabs:: + + .. group-tab:: bash + + .. code-block:: sh + + $ sudo -H useradd --shell /bin/bash --system \\ + --home-dir "$SERVICE_HOME" \\ + --comment "Privacy-respecting metasearch engine" $SERVICE_USER + + $ sudo -H mkdir "$SERVICE_HOME" + $ sudo -H chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME" + +.. END create user + +.. START clone searx + +.. tabs:: + + .. group-tab:: bash + + .. code-block:: sh + + $ sudo -H -u ${SERVICE_USER} -i + (${SERVICE_USER})$ git clone "https://github.com/asciimoo/searx.git" "$SEARX_SRC" + +.. END clone searx + +.. START create virtualenv + +.. tabs:: + + .. group-tab:: bash + + .. code-block:: sh + + (${SERVICE_USER})$ python3 -m venv "${SEARX_PYENV}" + (${SERVICE_USER})$ echo ". ${SEARX_PYENV}/bin/activate" >> "$SERVICE_HOME/.profile" + +.. END create virtualenv + +.. START manage.sh update_packages + +.. tabs:: + + .. group-tab:: bash + + .. code-block:: sh + + $ sudo -H -u ${SERVICE_USER} -i + + (${SERVICE_USER})$ command -v python && python --version + $SEARX_PYENV/bin/python + Python 3.8.1 + + # update pip's boilerplate .. + pip install -U pip + pip install -U setuptools + pip install -U wheel + + # jump to searx's working tree and install searx into virtualenv + (${SERVICE_USER})$ cd "$SEARX_SRC" + (${SERVICE_USER})$ pip install -e . + + +.. END manage.sh update_packages + +.. START searx config + +.. tabs:: + + .. group-tab:: bash + + .. code-block:: sh + + $ sudo -H cp "$SEARX_SRC/searx/settings.yml" "${SEARX_SETTINGS_PATH}" + $ sudo -H sed -i -e "s/ultrasecretkey/\\$(openssl rand -hex 16)/g" "$SEARX_SETTINGS_PATH" + $ sudo -H sed -i -e "s/{instance_name}/searx@\\$(uname -n)/g" "$SEARX_SETTINGS_PATH" + +.. END searx config + +.. START check searx installation + +.. tabs:: + + .. group-tab:: bash + + .. code-block:: sh + + # enable debug .. + $ sudo -H sed -i -e "s/debug : False/debug : True/g" "$SEARX_SETTINGS_PATH" + + # start webapp + $ sudo -H -u ${SERVICE_USER} -i + (${SERVICE_USER})$ cd ${SEARX_SRC} + (${SERVICE_USER})$ export SEARX_SETTINGS_PATH="${SEARX_SETTINGS_PATH}" + (${SERVICE_USER})$ python searx/webapp.py + + # disable debug + $ sudo -H sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS_PATH" + +Open WEB browser and visit http://$SEARX_INTERNAL_URL . If you are inside a +container or in a script, test with curl: + +.. tabs:: + + .. group-tab:: WEB browser + + .. code-block:: sh + + $ xgd-open http://$SEARX_INTERNAL_URL + + .. group-tab:: curl + + .. code-block:: none + + $ curl --location --verbose --head --insecure $SEARX_INTERNAL_URL + + * Trying 127.0.0.1:8888... + * TCP_NODELAY set + * Connected to 127.0.0.1 (127.0.0.1) port 8888 (#0) + > HEAD / HTTP/1.1 + > Host: 127.0.0.1:8888 + > User-Agent: curl/7.68.0 + > Accept: */* + > + * Mark bundle as not supporting multiuse + * HTTP 1.0, assume close after body + < HTTP/1.0 200 OK + HTTP/1.0 200 OK + ... + +.. END check searx installation diff --git a/docs/conf.py b/docs/conf.py index 03e977aba..1f79e4c14 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- import sys, os +from sphinx_build_tools import load_sphinx_config from searx.version import VERSION_STRING from pallets_sphinx_themes import ProjectLink from searx.brand import GIT_URL +GIT_BRANCH = os.environ.get("GIT_BRANCH", "master") from searx.brand import SEARX_URL from searx.brand import DOCS_URL @@ -22,6 +24,8 @@ master_doc = "index" source_suffix = '.rst' numfig = True +exclude_patterns = ['build-templates/*.rst'] + from searx import webapp jinja_contexts = { 'webapp': dict(**webapp.__dict__) @@ -35,7 +39,7 @@ extlinks['wiki'] = ('https://github.com/asciimoo/searx/wiki/%s', ' ') extlinks['pull'] = ('https://github.com/asciimoo/searx/pull/%s', 'PR ') # links to custom brand -extlinks['origin'] = (GIT_URL + '/blob/master/%s', 'git://') +extlinks['origin'] = (GIT_URL + '/blob/' + GIT_BRANCH + '/%s', 'git://') extlinks['patch'] = (GIT_URL + '/commit/%s', '#') extlinks['search'] = (SEARX_URL + '/%s', '#') extlinks['docs'] = (DOCS_URL + '/%s', 'docs: ') @@ -61,6 +65,8 @@ extensions = [ "pallets_sphinx_themes", "sphinx_issues", # https://github.com/sloria/sphinx-issues/blob/master/README.rst "sphinxcontrib.jinja", # https://github.com/tardyp/sphinx-jinja + "sphinxcontrib.programoutput", # https://github.com/NextThought/sphinxcontrib-programoutput + 'linuxdoc.kernel_include', # Implementation of the 'kernel-include' reST-directive. 'linuxdoc.rstFlatTable', # Implementation of the 'flat-table' reST-directive. 'linuxdoc.kfigure', # Sphinx extension which implements scalable image handling. "sphinx_tabs.tabs", # https://github.com/djungelorm/sphinx-tabs @@ -112,3 +118,9 @@ html_show_sourcelink = False latex_documents = [ (master_doc, "searx-{}.tex".format(VERSION_STRING), html_title, author, "manual") ] + +# ------------------------------------------------------------------------------ +# Since loadConfig overwrites settings from the global namespace, it has to be +# the last statement in the conf.py file +# ------------------------------------------------------------------------------ +load_sphinx_config(globals()) diff --git a/docs/dev/contribution_guide.rst b/docs/dev/contribution_guide.rst index 459dfb448..f9e9569f9 100644 --- a/docs/dev/contribution_guide.rst +++ b/docs/dev/contribution_guide.rst @@ -4,6 +4,11 @@ How to contribute ================= +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + Prime directives: Privacy, Hackability ====================================== diff --git a/docs/dev/index.rst b/docs/dev/index.rst index cb913a82b..ba0a25a9c 100644 --- a/docs/dev/index.rst +++ b/docs/dev/index.rst @@ -3,7 +3,8 @@ Developer documentation ======================= .. toctree:: - :maxdepth: 1 + :maxdepth: 2 + :caption: Contents quickstart contribution_guide diff --git a/docs/dev/makefile.rst b/docs/dev/makefile.rst index f5957001c..62cd0a984 100644 --- a/docs/dev/makefile.rst +++ b/docs/dev/makefile.rst @@ -11,23 +11,17 @@ Makefile Targets Before looking deeper at the targets, first read about :ref:`makefile setup` and :ref:`make pyenv`. + To install system requirements follow :ref:`buildhosts`. + With the aim to simplify development cycles, started with :pull:`1756` a ``Makefile`` based boilerplate was added. If you are not familiar with Makefiles, we recommend to read gnu-make_ introduction. The usage is simple, just type ``make {target-name}`` to *build* a target. -Calling the ``help`` target gives a first overview:: - - $ make help - test - run developer tests - docs - build documentation - docs-live - autobuild HTML documentation while editing - run - run developer instance - install - developer install (./local) - uninstall - uninstall (./local) - gh-pages - build docs & deploy on gh-pages branch - clean - drop builds and environments - ... +Calling the ``help`` target gives a first overview (``make help``): + +.. program-output:: bash -c "cd ..; make --no-print-directory help" + .. contents:: Contents :depth: 2 @@ -37,27 +31,33 @@ Calling the ``help`` target gives a first overview:: .. _makefile setup: -Setup -===== +Makefile setup +============== .. _git stash: https://git-scm.com/docs/git-stash -The main setup is done in the :origin:`Makefile`:: - - export GIT_URL=https://github.com/asciimoo/searx - export SEARX_URL=https://searx.me - export DOCS_URL=https://asciimoo.github.io/searx - .. sidebar:: fork & upstream Commit changes in your (local) branch, fork or whatever, but do not push them upstream / `git stash`_ is your friend. -:GIT_URL: Changes this, to point to your searx fork. +The main setup is done in the :origin:`Makefile`. + +.. literalinclude:: ../../Makefile + :start-after: START Makefile setup + :end-before: END Makefile setup + +:GIT_URL: Changes this, to point to your searx fork. +:GIT_BRANCH: Changes this, to point to your searx branch. +:SEARX_URL: Changes this, to point to your searx instance. +:DOCS_URL: If you host your own (*brand*) documentation, change this URL. -:SEARX_URL: Changes this, to point to your searx instance. +If you change any of this build environment variables, you have to run ``make +buildenv``:: -:DOCS_URL: If you host your own (branded) documentation, change this URL. + $ make buildenv + build searx/brand.py + build utils/brand.env .. _make pyenv: @@ -170,7 +170,7 @@ e.g.: .. code:: sh - $ make test.pep8 test.unit + $ make test.pep8 test.unit test.sh . ./local/py3/bin/activate; ./manage.sh pep8_check [!] Running pep8 check . ./local/py3/bin/activate; ./manage.sh unit_tests diff --git a/docs/dev/quickstart.rst b/docs/dev/quickstart.rst index a786eda63..a4fc897c4 100644 --- a/docs/dev/quickstart.rst +++ b/docs/dev/quickstart.rst @@ -27,7 +27,7 @@ searx-ve virtualenv and install the required packages using ``manage.sh``. cd ~/myprojects git clone https://github.com/asciimoo/searx.git cd searx - virtualenv searx-ve + python3 -m venv searx-ve . ./searx-ve/bin/activate ./manage.sh update_dev_packages diff --git a/docs/dev/reST.rst b/docs/dev/reST.rst index 4835cbbcf..906a0e9af 100644 --- a/docs/dev/reST.rst +++ b/docs/dev/reST.rst @@ -325,8 +325,9 @@ Literal blocks The simplest form of :duref:`literal-blocks` is a indented block introduced by two colons (``::``). For highlighting use :dudir:`highlight` or :ref:`reST -code` directive. To include literals from external files use directive -:dudir:`literalinclude`. +code` directive. To include literals from external files use +:rst:dir:`literalinclude` or :ref:`kernel-include <kernel-include-directive>` +directive (latter one expands environment variables in the path name). .. _reST literal: @@ -1312,9 +1313,8 @@ others are basic-tabs_ and code-tabs_. Below a *group-tab* example from .. literalinclude:: ../admin/buildhosts.rst :language: reST - :start-after: .. _system requirements: - :end-before: .. _system requirements END: - + :start-after: .. SNIP sh lint requirements + :end-before: .. SNAP sh lint requirements .. _math: diff --git a/docs/index.rst b/docs/index.rst index b62f4dc5d..9e590867c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,7 +2,14 @@ Welcome to searx ================ -Search without being tracked. + *Search without being tracked.* + +Searx is a free internet metasearch engine which aggregates results from more +than 70 search services. Users are neither tracked nor profiled. Additionally, +searx can be used over Tor for online anonymity. + +Get started with searx by using one of the Searx-instances_. If you don't trust +anyone, you can set up your own, see :ref:`installation`. .. sidebar:: Features @@ -16,19 +23,14 @@ Search without being tracked. - Hosted by organizations, such as *La Quadrature du Net*, which promote digital rights -Searx is a free internet metasearch engine which aggregates results from more -than 70 search services. Users are neither tracked nor profiled. Additionally, -searx can be used over Tor for online anonymity. - -Get started with searx by using one of the Searx-instances_. If you don't trust -anyone, you can set up your own, see :ref:`installation`. - .. toctree:: :maxdepth: 2 + :caption: Contents user/index admin/index dev/index + utils/index blog/index .. _Searx-instances: https://searx.space diff --git a/docs/user/conf.py b/docs/user/conf.py new file mode 100644 index 000000000..53ade4b63 --- /dev/null +++ b/docs/user/conf.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8; mode: python -*- +"""Configuration for the Searx user handbook +""" +project = 'Searx User-HB' +version = release = VERSION_STRING + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ('index' # startdocname + , 'searx-user-hb.tex' # targetname + , '' # take title from .rst + , author # author + , 'howto' # documentclass + , False # toctree_only + ), +] + diff --git a/docs/user/index.rst b/docs/user/index.rst index b13aca216..96d11bf8a 100644 --- a/docs/user/index.rst +++ b/docs/user/index.rst @@ -3,7 +3,8 @@ User documentation ================== .. toctree:: - :maxdepth: 1 + :maxdepth: 2 + :caption: Contents search_syntax own-instance diff --git a/docs/user/own-instance.rst b/docs/user/own-instance.rst index a2f736562..4876abe99 100644 --- a/docs/user/own-instance.rst +++ b/docs/user/own-instance.rst @@ -2,8 +2,10 @@ Why use a private instance? =========================== -"Is it worth to run my own instance?" is a common question among searx users. -Before answering this question, see what options a searx user has. + *"Is it worth to run my own instance?"* + +\.\. is a common question among searx users. Before answering this question, +see what options a searx user has. Public instances are open to everyone who has access to its URL. Usually, these are operated by unknown parties (from the users' point of view). Private diff --git a/docs/user/public_instances.rst b/docs/user/public_instances.rst deleted file mode 100644 index f58ab7a02..000000000 --- a/docs/user/public_instances.rst +++ /dev/null @@ -1,3 +0,0 @@ -:orphan: - -This page page has been moved to `searx.space <https://searx.space/>`__ diff --git a/docs/utils/filtron.sh.rst b/docs/utils/filtron.sh.rst new file mode 100644 index 000000000..86e3fa88a --- /dev/null +++ b/docs/utils/filtron.sh.rst @@ -0,0 +1,80 @@ + +.. _filtron.sh: + +==================== +``utils/filtron.sh`` +==================== + +.. sidebar:: further reading + + - :ref:`searx filtron` + - :ref:`architecture` + - :ref:`installation` (:ref:`nginx <installation nginx>` & :ref:`apache + <installation apache>`) + +.. _Go: https://golang.org/ +.. _filtron: https://github.com/asciimoo/filtron +.. _filtron README: https://github.com/asciimoo/filtron/blob/master/README.md + +To simplify installation and maintenance of a filtron instance you can use the +script :origin:`utils/filtron.sh`. In most cases you will install filtron_ +simply by running the command: + +.. code:: bash + + sudo -H ./utils/filtron.sh install all + +The script adds a ``${SERVICE_USER}`` (default:``filtron``) and installs filtron_ +into this user account: + +#. Create a separated user account (``filtron``). +#. Download and install Go_ binary in user's $HOME (``~filtron``). +#. Install filtron with the package management from Go_ (``go get -v -u + github.com/asciimoo/filtron``) +#. Setup a proper rule configuration :origin:`[ref] + <utils/templates/etc/filtron/rules.json>` (``/etc/filtron/rules.json``). +#. Setup a systemd service unit :origin:`[ref] + <utils/templates/lib/systemd/system/filtron.service>` + (``/lib/systemd/system/filtron.service``). + + +Create user +=========== + +.. kernel-include:: $DOCS_BUILD/includes/filtron.rst + :start-after: START create user + :end-before: END create user + + +Install go +========== + +.. kernel-include:: $DOCS_BUILD/includes/filtron.rst + :start-after: START install go + :end-before: END install go + + +Install filtron +=============== + +Install :origin:`rules.json <utils/templates/etc/filtron/rules.json>` at +``/etc/filtron/rules.json`` (see :ref:`Sample configuration of filtron`) and +install filtron software and systemd unit: + +.. kernel-include:: $DOCS_BUILD/includes/filtron.rst + :start-after: START install filtron + :end-before: END install filtron + +.. kernel-include:: $DOCS_BUILD/includes/filtron.rst + :start-after: START install systemd unit + :end-before: END install systemd unit + +.. _filtron.sh overview: + +Overview +======== + +The ``--help`` output of the script is largely self-explanatory +(:ref:`toolboxing common`): + +.. program-output:: ../utils/filtron.sh --help diff --git a/docs/utils/index.rst b/docs/utils/index.rst new file mode 100644 index 000000000..13914af28 --- /dev/null +++ b/docs/utils/index.rst @@ -0,0 +1,53 @@ +.. _searx_utils: +.. _toolboxing: + +======================= +Tooling box ``utils/*`` +======================= + +In the folder :origin:`utils/` we maintain some tools useful for admins and +developers. + +.. toctree:: + :maxdepth: 2 + :caption: Contents + + searx.sh + filtron.sh + morty.sh + lxc.sh + +.. _toolboxing common: + +Common commands & environment +============================= + +Scripts to maintain services often dispose of common commands and environments. + +``shell`` : command + Opens a shell from the service user ``${SERVICE_USSR}``, very helpful for + troubleshooting. + +``inspect service`` : command + Shows status and log of the service, most often you have a option to enable + more verbose debug logs. Very helpful for debugging, but be careful not to + enable debugging in a production environment! + +``FORCE_TIMEOUT`` : environment + Sets timeout for interactive prompts. If you want to run a script in batch + job, with defaults choices, set ``FORCE_TIMEOUT=0``. By example; to install a + reverse proxy for filtron on all containers of the :ref:`searx suite + <lxc-searx.env>` use :: + + sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/filtron.sh apache install + +.. _toolboxing setup: + +Tooling box setup +================= + +The main setup is done in the :origin:`.config.sh` (read also :ref:`makefile +setup`). + +.. literalinclude:: ../../.config.sh + :language: bash diff --git a/docs/utils/lxc.sh.rst b/docs/utils/lxc.sh.rst new file mode 100644 index 000000000..ae1412870 --- /dev/null +++ b/docs/utils/lxc.sh.rst @@ -0,0 +1,148 @@ + +.. _snap: https://snapcraft.io +.. _snapcraft LXD: https://snapcraft.io/lxd +.. _LXC/LXD Image Server: https://uk.images.linuxcontainers.org/ +.. _LXC: https://linuxcontainers.org/lxc/introduction/ +.. _LXD: https://linuxcontainers.org/lxd/introduction/ +.. _`LXD@github`: https://github.com/lxc/lxd + +.. _archlinux: https://www.archlinux.org/ + +.. _lxc.sh: + +================ +``utils/lxc.sh`` +================ + +.. sidebar:: further reading + + - snap_, `snapcraft LXD`_ + - LXC_, LXD_ + - `LXC/LXD Image Server`_ + - `LXD@github`_ + +With the use of *Linux Containers* (LXC_) we can scale our tasks over a stack of +containers, what we call the: *lxc suite*. The *searx suite* +(:origin:`lxc-searx.env <utils/lxc-searx.env>`) is loaded by default, every time +you start the ``lxc.sh`` script (*you do not need to care about*). + +Before you can start with containers, you need to install and initiate LXD_ +once:: + + $ snap install lxd + $ lxd init --auto + +To make use of the containers from the *searx suite*, you have to build the +:ref:`LXC suite containers <lxc.sh help>` initial. But be warned, **this might +take some time**:: + + $ sudo -H ./utils/lxc.sh build + +A cup of coffee later, your LXC suite is build up and you can run whatever task +you want / in a selected or even in all :ref:`LXC suite containers <lxc.sh +help>`. If you do not want to build all containers, **you can build just +one**:: + + $ sudo -H ./utils/lxc.sh build searx-ubu1804 + +*Good to know ...* + +Eeach container shares the root folder of the repository and the +command ``utils/lxc.sh cmd`` **handles relative path names transparent**, +compare output of:: + + $ sudo -H ./utils/lxc.sh cmd -- ls -la Makefile + ... + +In the containers, you can run what ever you want, e.g. to start a bash use:: + + $ sudo -H ./utils/lxc.sh cmd searx-ubu1804 bash + INFO: [searx-ubu1804] bash + root@searx-ubu1804:/share/searx# + +If there comes the time you want to **get rid off all** the containers and +**clean up local images** just type:: + + $ sudo -H ./utils/lxc.sh remove + $ sudo -H ./utils/lxc.sh remove images + + +Install suite +============= + +To install the complete :ref:`searx suite (includes searx, morty & filtron) +<lxc-searx.env>` into all LXC_ use:: + + $ sudo -H ./utils/lxc.sh install suite + +The command above installs a searx suite (see :ref:`installation scripts`). To +get the IP (URL) of the filtron service in the containers use ``show suite`` +command. To test instances from containers just open the URLs in your +WEB-Browser:: + + $ sudo ./utils/lxc.sh show suite | grep filtron + [searx-ubu1604] INFO: (eth0) filtron: http://n.n.n.246:4004/ http://n.n.n.246/searx + [searx-ubu1804] INFO: (eth0) filtron: http://n.n.n.147:4004/ http://n.n.n.147/searx + [searx-ubu1910] INFO: (eth0) filtron: http://n.n.n.140:4004/ http://n.n.n.140/searx + [searx-ubu2004] INFO: (eth0) filtron: http://n.n.n.18:4004/ http://n.n.n.18/searx + [searx-fedora31] INFO: (eth0) filtron: http://n.n.n.46:4004/ http://n.n.n.46/searx + [searx-archlinux] INFO: (eth0) filtron: http://n.n.n.32:4004/ http://n.n.n.32/searx + +To :ref:`install a nginx <installation nginx>` reverse proxy for filtron and +morty use (or alternatively use :ref:`apache <installation apache>`):: + + sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/filtron.sh nginx install + sudo -H ./utils/lxc.sh cmd -- FORCE_TIMEOUT=0 ./utils/morty.sh nginx install + + +Running commands +================ + +**Inside containers, you can use make or run scripts** from the +:ref:`toolboxing`. By example: to setup a :ref:`buildhosts` and run the +Makefile target ``test`` in the archlinux_ container:: + + sudo -H ./utils/lxc.sh cmd searx-archlinux ./utils/searx.sh install buildhost + sudo -H ./utils/lxc.sh cmd searx-archlinux make test + + +Setup searx buildhost +===================== + +You can **install the searx buildhost environment** into one or all containers. +The installation procedure to set up a :ref:`build host<buildhosts>` takes its +time. Installation in all containers will take more time (time for another cup +of coffee).:: + + sudo -H ./utils/lxc.sh cmd -- ./utils/searx.sh install buildhost + +To build (live) documentation inside a archlinux_ container:: + + sudo -H ./utils/lxc.sh cmd searx-archlinux make docs-clean docs-live + ... + [I 200331 15:00:42 server:296] Serving on http://0.0.0.0:8080 + +To get IP of the container and the port number *live docs* is listening:: + + $ sudo ./utils/lxc.sh show suite | grep docs-live + ... + [searx-archlinux] INFO: (eth0) docs-live: http://n.n.n.12:8080/ + + +.. _lxc.sh help: + +Overview +======== + +The ``--help`` output of the script is largely self-explanatory: + +.. program-output:: ../utils/lxc.sh --help + + +.. _lxc-searx.env: + +searx suite +=========== + +.. literalinclude:: ../../utils/lxc-searx.env + :language: bash diff --git a/docs/utils/morty.sh.rst b/docs/utils/morty.sh.rst new file mode 100644 index 000000000..3bdf9bdd3 --- /dev/null +++ b/docs/utils/morty.sh.rst @@ -0,0 +1,80 @@ + +.. _morty: https://github.com/asciimoo/morty +.. _morty's README: https://github.com/asciimoo/morty +.. _Go: https://golang.org/ + +.. _morty.sh: + +================== +``utils/morty.sh`` +================== + +.. sidebar:: further reading + + - :ref:`architecture` + - :ref:`installation` (:ref:`nginx <installation nginx>` & :ref:`apache + <installation apache>`) + - :ref:`searx morty` + +To simplify installation and maintenance of a morty_ instance you can use the +script :origin:`utils/morty.sh`. In most cases you will install morty_ simply by +running the command: + +.. code:: bash + + sudo -H ./utils/morty.sh install all + +The script adds a ``${SERVICE_USER}`` (default:``morty``) and installs morty_ +into this user account: + +#. Create a separated user account (``morty``). +#. Download and install Go_ binary in user's $HOME (``~morty``). +#. Install morty_ with the package management from Go_ (``go get -v -u + github.com/asciimoo/morty``) +#. Setup a systemd service unit :origin:`[ref] + <utils/templates/lib/systemd/system/morty.service>` + (``/lib/systemd/system/morty.service``). + +.. hint:: + + To add morty to your searx instance read chapter :ref:`searx morty`. + +Create user +=========== + +.. kernel-include:: $DOCS_BUILD/includes/morty.rst + :start-after: START create user + :end-before: END create user + + +Install go +========== + +.. kernel-include:: $DOCS_BUILD/includes/morty.rst + :start-after: START install go + :end-before: END install go + + +Install morty +============= + +Install morty software and systemd unit: + +.. kernel-include:: $DOCS_BUILD/includes/morty.rst + :start-after: START install morty + :end-before: END install morty + +.. kernel-include:: $DOCS_BUILD/includes/morty.rst + :start-after: START install systemd unit + :end-before: END install systemd unit + +.. _morty.sh overview: + +Overview +======== + +The ``--help`` output of the script is largely self-explanatory +(:ref:`toolboxing common`): + +.. program-output:: ../utils/morty.sh --help + diff --git a/docs/utils/searx.sh.rst b/docs/utils/searx.sh.rst new file mode 100644 index 000000000..dd4442f94 --- /dev/null +++ b/docs/utils/searx.sh.rst @@ -0,0 +1,39 @@ + +.. _searx.sh: + +================== +``utils/searx.sh`` +================== + +.. sidebar:: further reading + + - :ref:`architecture` + - :ref:`installation` + - :ref:`installation nginx` + - :ref:`installation apache` + +To simplify installation and maintenance of a searx instance you can use the +script :origin:`utils/searx.sh`. + +Install +======= + +In most cases you will install searx simply by running the command: + +.. code:: bash + + sudo -H ./utils/searx.sh install all + +The script adds a ``${SERVICE_USER}`` (default:``searx``) and installs searx +into this user account. The installation is described in chapter +:ref:`installation basic`. + +.. _intranet reverse proxy: + +Overview +======== + +The ``--help`` output of the script is largely self-explanatory +(:ref:`toolboxing common`): + +.. program-output:: ../utils/searx.sh --help diff --git a/requirements-dev.txt b/requirements-dev.txt index eed7ec3d2..5a1ca1599 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,3 +15,5 @@ selenium==3.141.0 linuxdoc @ git+http://github.com/return42/linuxdoc.git sphinx-jinja sphinx-tabs +sphinxcontrib-programoutput +twine diff --git a/searx/brand.py b/searx/brand.py index d563447f6..91d2ab321 100644 --- a/searx/brand.py +++ b/searx/brand.py @@ -1,4 +1,5 @@ GIT_URL = 'https://github.com/asciimoo/searx' +GIT_BRANCH = 'master' ISSUE_URL = 'https://github.com/asciimoo/searx/issues' SEARX_URL = 'https://searx.me' DOCS_URL = 'https://asciimoo.github.io/searx' diff --git a/searx/static/plugins/js/search_on_category_select.js b/searx/static/plugins/js/search_on_category_select.js index 1c42d9e9e..d590ed127 100644 --- a/searx/static/plugins/js/search_on_category_select.js +++ b/searx/static/plugins/js/search_on_category_select.js @@ -6,19 +6,37 @@ $(document).ready(function() { }); $(document.getElementById($(this).attr("for"))).prop('checked', true); if($('#q').val()) { + if (getHttpRequest() == "GET") { + $('#search_form').attr('action', $('#search_form').serialize()); + } $('#search_form').submit(); } return false; }); $('#time-range').change(function(e) { if($('#q').val()) { + if (getHttpRequest() == "GET") { + $('#search_form').attr('action', $('#search_form').serialize()); + } $('#search_form').submit(); } }); $('#language').change(function(e) { if($('#q').val()) { + if (getHttpRequest() == "GET") { + $('#search_form').attr('action', $('#search_form').serialize()); + } $('#search_form').submit(); } }); } }); + +function getHttpRequest() { + httpRequest = "POST"; + urlParams = new URLSearchParams(window.location.search); + if (urlParams.has('method')) { + httpRequest = urlParams.get('method'); + } + return httpRequest; +} diff --git a/searx/templates/__common__/opensearch.xml b/searx/templates/__common__/opensearch.xml index 15d3eb792..27634245f 100644 --- a/searx/templates/__common__/opensearch.xml +++ b/searx/templates/__common__/opensearch.xml @@ -6,23 +6,13 @@ <Image>{{ urljoin(host, url_for('static', filename='img/favicon.png')) }}</Image> <LongName>searx metasearch</LongName> {% if opensearch_method == 'get' %} - <Url type="text/html" method="get" template="{{ host }}search?q={searchTerms}"/> - {% if autocomplete %} - <Url type="application/x-suggestions+json" method="get" template="{{ host }}autocompleter"> - <Param name="format" value="x-suggestions" /> - <Param name="q" value="{searchTerms}" /> - </Url> - {% endif %} + <Url rel="results" type="text/html" method="get" template="{{ host }}search?q={searchTerms}"/> {% else %} - <Url type="text/html" method="post" template="{{ host }}"> + <Url rel="results" type="text/html" method="post" template="{{ host }}"> <Param name="q" value="{searchTerms}" /> </Url> - {% if autocomplete %} - <!-- TODO, POST REQUEST doesn't work --> - <Url type="application/x-suggestions+json" method="get" template="{{ host }}autocompleter"> - <Param name="format" value="x-suggestions" /> - <Param name="q" value="{searchTerms}" /> - </Url> - {% endif %} + {% endif %} + {% if autocomplete %} + <Url rel="suggestions" type="application/json" template="{{ host }}autocompleter"/> {% endif %} </OpenSearchDescription> diff --git a/searx/webapp.py b/searx/webapp.py index 68be25584..35495a0ff 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -949,7 +949,7 @@ def opensearch(): resp = Response(response=ret, status=200, - mimetype="text/xml") + mimetype="application/opensearchdescription+xml") return resp diff --git a/utils/brand.env b/utils/brand.env index 7fe1a3911..5d5b39513 100644 --- a/utils/brand.env +++ b/utils/brand.env @@ -1,4 +1,5 @@ export GIT_URL='https://github.com/asciimoo/searx' +export GIT_BRANCH='master' export ISSUE_URL='https://github.com/asciimoo/searx/issues' export SEARX_URL='https://searx.me' export DOCS_URL='https://asciimoo.github.io/searx' diff --git a/utils/filtron.sh b/utils/filtron.sh new file mode 100755 index 000000000..8986fb0ef --- /dev/null +++ b/utils/filtron.sh @@ -0,0 +1,561 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- +# SPDX-License-Identifier: AGPL-3.0-or-later +# shellcheck disable=SC2119,SC2001 + +# shellcheck source=utils/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +# shellcheck source=utils/brand.env +source "${REPO_ROOT}/utils/brand.env" +source_dot_config +source "${REPO_ROOT}/utils/lxc-searx.env" +in_container && lxc_set_suite_env + +# ---------------------------------------------------------------------------- +# config +# ---------------------------------------------------------------------------- + +PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}" +PUBLIC_HOST="${PUBLIC_HOST:-$(echo "$PUBLIC_URL" | sed -e 's/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/')}" + +FILTRON_URL_PATH="${FILTRON_URL_PATH:-$(echo "${PUBLIC_URL}" \ +| sed -e 's,^.*://[^/]*\(/.*\),\1,g')}" +[[ "${FILTRON_URL_PATH}" == "${PUBLIC_URL}" ]] && FILTRON_URL_PATH=/ + +FILTRON_ETC="/etc/filtron" +FILTRON_RULES="$FILTRON_ETC/rules.json" + +FILTRON_API="${FILTRON_API:-127.0.0.1:4005}" +FILTRON_LISTEN="${FILTRON_LISTEN:-127.0.0.1:4004}" +FILTRON_TARGET="${FILTRON_TARGET:-127.0.0.1:8888}" + +SERVICE_NAME="filtron" +SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}" +SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}" +SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}" +SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service" +# shellcheck disable=SC2034 +SERVICE_GROUP="${SERVICE_USER}" + +# shellcheck disable=SC2034 +SERVICE_GROUP="${SERVICE_USER}" + +GO_ENV="${SERVICE_HOME}/.go_env" +GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz" +GO_TAR=$(basename "$GO_PKG_URL") + +APACHE_FILTRON_SITE="searx.conf" +NGINX_FILTRON_SITE="searx.conf" + +# shellcheck disable=SC2034 +CONFIG_FILES=( + "${FILTRON_RULES}" + "${SERVICE_SYSTEMD_UNIT}" +) + +# ---------------------------------------------------------------------------- +usage() { +# ---------------------------------------------------------------------------- + + # shellcheck disable=SC1117 + cat <<EOF +usage:: + $(basename "$0") shell + $(basename "$0") install [all|user|rules] + $(basename "$0") update [filtron] + $(basename "$0") remove [all] + $(basename "$0") activate [service] + $(basename "$0") deactivate [service] + $(basename "$0") inspect [service] + $(basename "$0") option [debug-on|debug-off] + $(basename "$0") apache [install|remove] + $(basename "$0") nginx [install|remove] + +shell + start interactive shell from user ${SERVICE_USER} +install / remove + :all: complete setup of filtron service + :user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME) + :rules: reinstall filtron rules $FILTRON_RULES +update filtron + Update filtron installation ($SERVICE_HOME) +activate service + activate and start service daemon (systemd unit) +deactivate service + stop and deactivate service daemon (systemd unit) +inspect service + show service status and log +option + set one of the available options +apache (${PUBLIC_URL}) + :install: apache site with a reverse proxy (ProxyPass) + :remove: apache site ${APACHE_FILTRON_SITE} +nginx (${PUBLIC_URL}) + :install: nginx site with a reverse proxy (ProxyPass) + :remove: nginx site ${NGINX_FILTRON_SITE} + +filtron rules: ${FILTRON_RULES} + +If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/"}' file:: + PUBLIC_URL : ${PUBLIC_URL} + PUBLIC_HOST : ${PUBLIC_HOST} + SERVICE_USER : ${SERVICE_USER} + FILTRON_TARGET : ${FILTRON_TARGET} + FILTRON_API : ${FILTRON_API} + FILTRON_LISTEN : ${FILTRON_LISTEN} +EOF + if in_container; then + # in containers the service is listening on 0.0.0.0 (see lxc-searx.env) + for ip in $(global_IPs) ; do + if [[ $ip =~ .*:.* ]]; then + echo " container URL (IPv6): http://[${ip#*|}]:4005/" + else + # IPv4: + echo " container URL (IPv4): http://${ip#*|}:4005/" + fi + done + fi + [[ -n ${1} ]] && err_msg "$1" +} + +main() { + required_commands \ + sudo install git wget curl \ + || exit + + local _usage="unknown or missing $1 command $2" + + case $1 in + --getenv) var="$2"; echo "${!var}"; exit 0;; + -h|--help) usage; exit 0;; + + shell) + sudo_or_exit + interactive_shell "${SERVICE_USER}" + ;; + inspect) + case $2 in + service) + sudo_or_exit + inspect_service + ;; + *) usage "$_usage"; exit 42;; + esac ;; + install) + rst_title "$SERVICE_NAME" part + sudo_or_exit + case $2 in + all) install_all ;; + user) assert_user ;; + rules) + rst_title "Re-Install filtron rules" + echo + install_template --no-eval "$FILTRON_RULES" root root 644 + systemd_restart_service "${SERVICE_NAME}" + ;; + *) usage "$_usage"; exit 42;; + esac ;; + update) + sudo_or_exit + case $2 in + filtron) update_filtron ;; + *) usage "$_usage"; exit 42;; + esac ;; + remove) + sudo_or_exit + case $2 in + all) remove_all;; + user) drop_service_account "${SERVICE_USER}" ;; + *) usage "$_usage"; exit 42;; + esac ;; + activate) + sudo_or_exit + case $2 in + service) systemd_activate_service "${SERVICE_NAME}" ;; + *) usage "$_usage"; exit 42;; + esac ;; + deactivate) + sudo_or_exit + case $2 in + service) systemd_deactivate_service "${SERVICE_NAME}" ;; + *) usage "$_usage"; exit 42;; + esac ;; + apache) + sudo_or_exit + case $2 in + install) install_apache_site ;; + remove) remove_apache_site ;; + *) usage "$_usage"; exit 42;; + esac ;; + nginx) + sudo_or_exit + case $2 in + install) install_nginx_site ;; + remove) remove_nginx_site ;; + *) usage "$_usage"; exit 42;; + esac ;; + option) + sudo_or_exit + case $2 in + debug-on) echo; enable_debug ;; + debug-off) echo; disable_debug ;; + *) usage "$_usage"; exit 42;; + esac ;; + doc) rst-doc ;; + *) usage "unknown or missing command $1"; exit 42;; + esac +} + +install_all() { + rst_title "Install $SERVICE_NAME (service)" + assert_user + wait_key + install_go "${GO_PKG_URL}" "${GO_TAR}" "${SERVICE_USER}" + wait_key + install_filtron + wait_key + systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" + wait_key + echo + if ! service_is_available "http://${FILTRON_LISTEN}" ; then + err_msg "Filtron does not listening on: http://${FILTRON_LISTEN}" + fi + if apache_is_installed; then + info_msg "Apache is installed on this host." + if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then + install_apache_site + fi + elif nginx_is_installed; then + info_msg "nginx is installed on this host." + if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then + install_nginx_site + fi + fi + if ask_yn "Do you want to inspect the installation?" Ny; then + inspect_service + fi + +} + +remove_all() { + rst_title "De-Install $SERVICE_NAME (service)" + + rst_para "\ +It goes without saying that this script can only be used to remove +installations that were installed with this script." + + if ! systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then + return 42 + fi + drop_service_account "${SERVICE_USER}" + rm -r "$FILTRON_ETC" 2>&1 | prefix_stdout + if service_is_available "${PUBLIC_URL}"; then + MSG="** Don't forget to remove your public site! (${PUBLIC_URL}) **" wait_key 10 + fi +} + +assert_user() { + rst_title "user $SERVICE_USER" section + echo + tee_stderr 1 <<EOF | bash | prefix_stdout +useradd --shell /bin/bash --system \ + --home-dir "$SERVICE_HOME" \ + --comment 'Reverse HTTP proxy to filter requests' $SERVICE_USER +mkdir "$SERVICE_HOME" +chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME" +groups $SERVICE_USER +EOF + SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)" + export SERVICE_HOME + echo "export SERVICE_HOME=$SERVICE_HOME" + + cat > "$GO_ENV" <<EOF +export GOPATH=\$HOME/go-apps +export PATH=\$PATH:\$HOME/local/go/bin:\$GOPATH/bin +EOF + echo "Environment $GO_ENV has been setup." + + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" +grep -qFs -- 'source $GO_ENV' ~/.profile || echo 'source $GO_ENV' >> ~/.profile +EOF +} + +filtron_is_installed() { + [[ -f $SERVICE_HOME/go-apps/bin/filtron ]] +} + +_svcpr=" ${_Yellow}|${SERVICE_USER}|${_creset} " + +install_filtron() { + rst_title "Install filtron in user's ~/go-apps" section + echo + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr" +go get -v -u github.com/asciimoo/filtron +EOF + install_template --no-eval "$FILTRON_RULES" root root 644 +} + +update_filtron() { + rst_title "Update filtron" section + echo + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr" +go get -v -u github.com/asciimoo/filtron +EOF +} + +inspect_service() { + + rst_title "service status & log" + + cat <<EOF + +sourced ${DOT_CONFIG#"$REPO_ROOT/"} : + + PUBLIC_URL : ${PUBLIC_URL} + PUBLIC_HOST : ${PUBLIC_HOST} + FILTRON_URL_PATH : ${FILTRON_URL_PATH} + FILTRON_API : ${FILTRON_API} + FILTRON_LISTEN : ${FILTRON_LISTEN} + FILTRON_TARGET : ${FILTRON_TARGET} + +EOF + + if service_account_is_available "$SERVICE_USER"; then + info_msg "service account $SERVICE_USER available." + else + err_msg "service account $SERVICE_USER not available!" + fi + if go_is_available "$SERVICE_USER"; then + info_msg "~$SERVICE_USER: go is installed" + else + err_msg "~$SERVICE_USER: go is not installed" + fi + if filtron_is_installed; then + info_msg "~$SERVICE_USER: filtron app is installed" + else + err_msg "~$SERVICE_USER: filtron app is not installed!" + fi + + if ! service_is_available "http://${FILTRON_API}"; then + err_msg "API not available at: http://${FILTRON_API}" + fi + + if ! service_is_available "http://${FILTRON_LISTEN}" ; then + err_msg "Filtron does not listening on: http://${FILTRON_LISTEN}" + fi + + if service_is_available "http://${FILTRON_TARGET}" ; then + info_msg "Filtron's target is available at: http://${FILTRON_TARGET}" + fi + + if ! service_is_available "${PUBLIC_URL}"; then + warn_msg "Public service at ${PUBLIC_URL} is not available!" + if ! in_container; then + warn_msg "Check if public name is correct and routed or use the public IP from above." + fi + fi + + if in_container; then + lxc_suite_info + else + info_msg "public URL --> ${PUBLIC_URL}" + info_msg "internal URL --> http://${FILTRON_LISTEN}" + fi + + + local _debug_on + if ask_yn "Enable filtron debug mode?"; then + enable_debug + _debug_on=1 + fi + echo + systemctl --no-pager -l status "${SERVICE_NAME}" + echo + + info_msg "public URL --> ${PUBLIC_URL}" + # shellcheck disable=SC2059 + printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log" + read -r -s -n1 -t 5 + echo + while true; do + trap break 2 + journalctl -f -u "${SERVICE_NAME}" + done + + if [[ $_debug_on == 1 ]]; then + disable_debug + fi + return 0 +} + + +enable_debug() { + info_msg "try to enable debug mode ..." + python <<EOF +import sys, json + +debug = { + u'name': u'debug request' + , u'filters': [] + , u'interval': 0 + , u'limit': 0 + , u'actions': [{u'name': u'log'}] +} + +with open('$FILTRON_RULES') as rules: + j = json.load(rules) + +pos = None +for i in range(len(j)): + if j[i].get('name') == 'debug request': + pos = i + break +if pos is not None: + j[pos] = debug +else: + j.append(debug) +with open('$FILTRON_RULES', 'w') as rules: + json.dump(j, rules, indent=2, sort_keys=True) + +EOF + systemctl restart "${SERVICE_NAME}.service" +} + +disable_debug() { + info_msg "try to disable debug mode ..." + python <<EOF +import sys, json +with open('$FILTRON_RULES') as rules: + j = json.load(rules) + +pos = None +for i in range(len(j)): + if j[i].get('name') == 'debug request': + pos = i + break +if pos is not None: + del j[pos] + with open('$FILTRON_RULES', 'w') as rules: + json.dump(j, rules, indent=2, sort_keys=True) +EOF + systemctl restart "${SERVICE_NAME}.service" +} + +install_apache_site() { + + rst_title "Install Apache site $APACHE_FILTRON_SITE" + + rst_para "\ +This installs a reverse proxy (ProxyPass) into apache site (${APACHE_FILTRON_SITE})" + + ! apache_is_installed && info_msg "Apache is not installed." + + if ! ask_yn "Do you really want to continue?" Yn; then + return + else + install_apache + fi + + "${REPO_ROOT}/utils/searx.sh" install uwsgi + + apache_install_site --variant=filtron "${APACHE_FILTRON_SITE}" + + info_msg "testing public url .." + if ! service_is_available "${PUBLIC_URL}"; then + err_msg "Public service at ${PUBLIC_URL} is not available!" + fi +} + +remove_apache_site() { + + rst_title "Remove Apache site $APACHE_FILTRON_SITE" + + rst_para "\ +This removes apache site ${APACHE_FILTRON_SITE}." + + ! apache_is_installed && err_msg "Apache is not installed." + + if ! ask_yn "Do you really want to continue?" Yn; then + return + fi + + apache_remove_site "$APACHE_FILTRON_SITE" + +} + +install_nginx_site() { + + rst_title "Install nginx site $NGINX_FILTRON_SITE" + + rst_para "\ +This installs a reverse proxy (ProxyPass) into nginx site (${NGINX_FILTRON_SITE})" + + ! nginx_is_installed && info_msg "nginx is not installed." + + if ! ask_yn "Do you really want to continue?" Yn; then + return + else + install_nginx + fi + + "${REPO_ROOT}/utils/searx.sh" install uwsgi + + # shellcheck disable=SC2034 + SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC) + # shellcheck disable=SC2034 + SEARX_URL_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_URL_PATH) + nginx_install_app --variant=filtron "${NGINX_FILTRON_SITE}" + + info_msg "testing public url .." + if ! service_is_available "${PUBLIC_URL}"; then + err_msg "Public service at ${PUBLIC_URL} is not available!" + fi +} + +remove_nginx_site() { + + rst_title "Remove nginx site $NGINX_FILTRON_SITE" + + rst_para "\ +This removes nginx site ${NGINX_FILTRON_SITE}." + + ! nginx_is_installed && err_msg "nginx is not installed." + + if ! ask_yn "Do you really want to continue?" Yn; then + return + fi + + nginx_remove_site "$FILTRON_FILTRON_SITE" + +} + + +rst-doc() { + + eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/filtron.rst")\"" + + echo -e "\n.. START install systemd unit" + cat <<EOF +.. tabs:: + + .. group-tab:: systemd + + .. code:: bash + +EOF + eval "echo \"$(< "${TEMPLATES}/${SERVICE_SYSTEMD_UNIT}")\"" | prefix_stdout " " + echo -e "\n.. END install systemd unit" + + # for DIST_NAME in ubuntu-20.04 arch fedora; do + # ( + # DIST_ID=${DIST_NAME%-*} + # DIST_VERS=${DIST_NAME#*-} + # [[ $DIST_VERS =~ $DIST_ID ]] && DIST_VERS= + # # ... + # ) + # done +} + +# ---------------------------------------------------------------------------- +main "$@" +# ---------------------------------------------------------------------------- diff --git a/utils/lib.sh b/utils/lib.sh new file mode 100755 index 000000000..922227a89 --- /dev/null +++ b/utils/lib.sh @@ -0,0 +1,1519 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- +# SPDX-License-Identifier: AGPL-3.0-or-later +# shellcheck disable=SC2059,SC1117 + +# ubuntu, debian, arch, fedora ... +DIST_ID=$(source /etc/os-release; echo "$ID"); +# shellcheck disable=SC2034 +DIST_VERS=$(source /etc/os-release; echo "$VERSION_ID"); + +ADMIN_NAME="${ADMIN_NAME:-$(git config user.name)}" +ADMIN_NAME="${ADMIN_NAME:-$USER}" + +ADMIN_EMAIL="${ADMIN_EMAIL:-$(git config user.email)}" +ADMIN_EMAIL="${ADMIN_EMAIL:-$USER@$(hostname)}" + +if [[ -z "${REPO_ROOT}" ]]; then + REPO_ROOT=$(dirname "${BASH_SOURCE[0]}") + while [ -h "${REPO_ROOT}" ] ; do + REPO_ROOT=$(readlink "${REPO_ROOT}") + done + REPO_ROOT=$(cd "${REPO_ROOT}/.." && pwd -P ) +fi + +if [[ -z ${TEMPLATES} ]]; then + TEMPLATES="${REPO_ROOT}/utils/templates" +fi + +if [[ -z "$CACHE" ]]; then + CACHE="${REPO_ROOT}/cache" +fi + +if [[ -z ${DIFF_CMD} ]]; then + DIFF_CMD="diff -u" + if command -v colordiff >/dev/null; then + DIFF_CMD="colordiff -u" + fi +fi + +DOT_CONFIG="${DOT_CONFIG:-${REPO_ROOT}/.config.sh}" + +source_dot_config() { + if [[ ! -e "${DOT_CONFIG}" ]]; then + err_msg "configuration does not extsts at: ${DOT_CONFIG}" + return 42 + fi + # shellcheck disable=SC1090 + source "${DOT_CONFIG}" +} + +sudo_or_exit() { + # usage: sudo_or_exit + + if [ ! "$(id -u)" -eq 0 ]; then + err_msg "this command requires root (sudo) privilege!" >&2 + exit 42 + fi +} + +required_commands() { + + # usage: required_commands [cmd1 ...] + + local exit_val=0 + while [ -n "$1" ]; do + + if ! command -v "$1" &>/dev/null; then + err_msg "missing command $1" + exit_val=42 + fi + shift + done + return $exit_val +} + +# colors +# ------ + +# shellcheck disable=SC2034 +set_terminal_colors() { + _colors=8 + _creset='\e[0m' # reset all attributes + + _Black='\e[0;30m' + _White='\e[1;37m' + _Red='\e[0;31m' + _Green='\e[0;32m' + _Yellow='\e[0;33m' + _Blue='\e[0;34m' + _Violet='\e[0;35m' + _Cyan='\e[0;36m' + + _BBlack='\e[1;30m' + _BWhite='\e[1;37m' + _BRed='\e[1;31m' + _BGreen='\e[1;32m' + _BYellow='\e[1;33m' + _BBlue='\e[1;34m' + _BPurple='\e[1;35m' + _BCyan='\e[1;36m' +} + +if [ ! -p /dev/stdout ]; then + set_terminal_colors +fi + +# reST +# ---- + +if command -v fmt >/dev/null; then + export FMT="fmt -u" +else + export FMT="cat" +fi + +rst_title() { + # usage: rst_title <header-text> [part|chapter|section] + + case ${2-chapter} in + part) printf "\n${_BGreen}${1//?/=}${_creset}\n${_BCyan}${1}${_creset}\n${_BGreen}${1//?/=}${_creset}\n";; + chapter) printf "\n${_BCyan}${1}${_creset}\n${_BGreen}${1//?/=}${_creset}\n";; + section) printf "\n${_BCyan}${1}${_creset}\n${_BGreen}${1//?/-}${_creset}\n";; + *) + err_msg "invalid argument '${2}' in line $(caller)" + return 42 + ;; + esac +} + +rst_para() { + # usage: RST_INDENT=1 rst_para "lorem ipsum ..." + local prefix='' + if [[ -n $RST_INDENT ]] && [[ $RST_INDENT -gt 0 ]]; then + prefix="$(for i in $(seq 1 "$RST_INDENT"); do printf " "; done)" + echo -en "\n$*\n" | $FMT | prefix_stdout "$prefix" + else + echo -en "\n$*\n" | $FMT + fi +} + +die() { + echo -e "${_BRed}ERROR:${_creset} ${BASH_SOURCE[1]}: line ${BASH_LINENO[0]}: ${2-died ${1-1}}" >&2; + exit "${1-1}" +} + +die_caller() { + echo -e "${_BRed}ERROR:${_creset} ${BASH_SOURCE[2]}: line ${BASH_LINENO[1]}: ${FUNCNAME[1]}(): ${2-died ${1-1}}" >&2; + exit "${1-1}" +} + +err_msg() { echo -e "${_BRed}ERROR:${_creset} $*" >&2; } +warn_msg() { echo -e "${_BBlue}WARN:${_creset} $*" >&2; } +info_msg() { echo -e "${_BYellow}INFO:${_creset} $*" >&2; } + +clean_stdin() { + if [[ $(uname -s) != 'Darwin' ]]; then + while read -r -n1 -t 0.1; do : ; done + fi +} + +wait_key(){ + # usage: waitKEY [<timeout in sec>] + + clean_stdin + local _t=$1 + local msg="${MSG}" + [[ -z "$msg" ]] && msg="${_Green}** press any [${_BCyan}KEY${_Green}] to continue **${_creset}" + + [[ -n $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT + [[ -n $_t ]] && _t="-t $_t" + printf "$msg" + # shellcheck disable=SC2086 + read -r -s -n1 $_t + echo + clean_stdin +} + +ask_yn() { + # usage: ask_yn <prompt-text> [Ny|Yn] [<timeout in sec>] + + local EXIT_YES=0 # exit status 0 --> successful + local EXIT_NO=1 # exit status 1 --> error code + + local _t=$3 + [[ -n $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT + [[ -n $_t ]] && _t="-t $_t" + case "${FORCE_SELECTION:-${2}}" in + Y) return ${EXIT_YES} ;; + N) return ${EXIT_NO} ;; + Yn) + local exit_val=${EXIT_YES} + local choice="[${_BGreen}YES${_creset}/no]" + local default="Yes" + ;; + *) + local exit_val=${EXIT_NO} + local choice="[${_BGreen}NO${_creset}/yes]" + local default="No" + ;; + esac + echo + while true; do + clean_stdin + printf "$1 ${choice} " + # shellcheck disable=SC2086 + read -r -n1 $_t + if [[ -z $REPLY ]]; then + printf "$default\n"; break + elif [[ $REPLY =~ ^[Yy]$ ]]; then + exit_val=${EXIT_YES} + printf "\n" + break + elif [[ $REPLY =~ ^[Nn]$ ]]; then + exit_val=${EXIT_NO} + printf "\n" + break + fi + _t="" + err_msg "invalid choice" + done + clean_stdin + return $exit_val +} + +tee_stderr () { + + # usage:: + # tee_stderr 1 <<EOF | python -i + # print("hello") + # EOF + # ... + # >>> print("hello") + # hello + + local _t="0"; + if [[ -n $1 ]] ; then _t="$1"; fi + + (while read -r line; do + # shellcheck disable=SC2086 + sleep $_t + echo -e "$line" >&2 + echo "$line" + done) +} + +prefix_stdout () { + # usage: <cmd> | prefix_stdout [prefix] + + local prefix="${_BYellow}-->|${_creset}" + + if [[ -n $1 ]] ; then prefix="$1"; fi + + # shellcheck disable=SC2162 + (while IFS= read line; do + echo -e "${prefix}$line" + done) +} + +append_line() { + + # usage: append_line <line> <file> + # + # Append line if not exists, create file if not exists. E.g:: + # + # append_line 'source ~/.foo' ~/bashrc + + local LINE=$1 + local FILE=$2 + grep -qFs -- "$LINE" "$FILE" || echo "$LINE" >> "$FILE" +} + +cache_download() { + + # usage: cache_download <url> <local-filename> + + local exit_value=0 + + if [[ -n ${SUDO_USER} ]]; then + sudo -u "${SUDO_USER}" mkdir -p "${CACHE}" + else + mkdir -p "${CACHE}" + fi + + if [[ -f "${CACHE}/$2" ]] ; then + info_msg "already cached: $1" + info_msg " --> ${CACHE}/$2" + fi + + if [[ ! -f "${CACHE}/$2" ]]; then + info_msg "caching: $1" + info_msg " --> ${CACHE}/$2" + if [[ -n ${SUDO_USER} ]]; then + sudo -u "${SUDO_USER}" wget --progress=bar -O "${CACHE}/$2" "$1" ; exit_value=$? + else + wget --progress=bar -O "${CACHE}/$2" "$1" ; exit_value=$? + fi + if [[ ! $exit_value = 0 ]]; then + err_msg "failed to download: $1" + fi + fi +} + +backup_file() { + + # usage: backup_file /path/to/file.foo + + local stamp + stamp=$(date +"_%Y%m%d_%H%M%S") + info_msg "create backup: ${1}${stamp}" + cp -a "${1}" "${1}${stamp}" +} + +choose_one() { + + # usage: + # + # DEFAULT_SELECT= 2 \ + # choose_one <name> "your selection?" "Coffee" "Coffee with milk" + + local default=${DEFAULT_SELECT-1} + local REPLY + local env_name=$1 && shift + local choice=$1; + local max="${#@}" + local _t + [[ -n $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT + [[ -n $_t ]] && _t="-t $_t" + + list=("$@") + echo -e "${_BGreen}Menu::${_creset}" + for ((i=1; i<= $((max -1)); i++)); do + if [[ "$i" == "$default" ]]; then + echo -e " ${_BGreen}$i.${_creset}) ${list[$i]} [default]" + else + echo -e " $i.) ${list[$i]}" + fi + done + while true; do + clean_stdin + printf "$1 [${_BGreen}$default${_creset}] " + + if (( 10 > max )); then + # shellcheck disable=SC2086 + read -r -n1 $_t + else + # shellcheck disable=SC2086,SC2229 + read -r $_t + fi + # selection fits + [[ $REPLY =~ ^-?[0-9]+$ ]] && (( REPLY > 0 )) && (( REPLY < max )) && break + + # take default + [[ -z $REPLY ]] && REPLY=$default && break + + _t="" + err_msg "invalid choice" + done + eval "$env_name"='${list[${REPLY}]}' + echo + clean_stdin +} + +install_template() { + + # usage: + # + # install_template [--no-eval] [--variant=<name>] \ + # {file} [{owner} [{group} [{chmod}]]] + # + # E.g. the origin of variant 'raw' of /etc/updatedb.conf is:: + # + # ${TEMPLATES}/etc/updatedb.conf:raw + # + # To install variant 'raw' of /etc/updatedb.conf without evaluated + # replacements you can use:: + # + # install_template --variant=raw --no-eval \ + # /etc/updatedb.conf root root 644 + + local _reply="" + local do_eval=1 + local variant="" + local pos_args=("$0") + + for i in "$@"; do + case $i in + --no-eval) do_eval=0; shift ;; + --variant=*) variant=":${i#*=}"; shift ;; + *) pos_args+=("$i") ;; + esac + done + + local dst="${pos_args[1]}" + local template_origin="${TEMPLATES}${dst}${variant}" + local template_file="${TEMPLATES}${dst}" + + local owner="${pos_args[2]-$(id -un)}" + local group="${pos_args[3]-$(id -gn)}" + local chmod="${pos_args[4]-644}" + + info_msg "install (eval=$do_eval): ${dst}" + [[ -n $variant ]] && info_msg "variant --> ${variant}" + + if [[ ! -f "${template_origin}" ]] ; then + err_msg "${template_origin} does not exists" + err_msg "... can't install $dst" + wait_key 30 + return 42 + fi + + if [[ "$do_eval" == "1" ]]; then + template_file="${CACHE}${dst}${variant}" + info_msg "BUILD template ${template_file}" + if [[ -n ${SUDO_USER} ]]; then + sudo -u "${SUDO_USER}" mkdir -p "$(dirname "${template_file}")" + else + mkdir -p "$(dirname "${template_file}")" + fi + # shellcheck disable=SC2086 + eval "echo \"$(cat ${template_origin})\"" > "${template_file}" + if [[ -n ${SUDO_USER} ]]; then + chown "${SUDO_USER}:${SUDO_USER}" "${template_file}" + fi + else + template_file=$template_origin + fi + + mkdir -p "$(dirname "${dst}")" + + if [[ ! -f "${dst}" ]]; then + info_msg "install: ${template_file}" + sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \ + "${template_file}" "${dst}" | prefix_stdout + return $? + fi + + if [[ -f "${dst}" ]] && cmp --silent "${template_file}" "${dst}" ; then + info_msg "file ${dst} allready installed" + return 0 + fi + + info_msg "diffrent file ${dst} allready exists on this host" + + while true; do + choose_one _reply "choose next step with file $dst" \ + "replace file" \ + "leave file unchanged" \ + "interactiv shell" \ + "diff files" + + case $_reply in + "replace file") + info_msg "install: ${template_file}" + sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \ + "${template_file}" "${dst}" | prefix_stdout + break + ;; + "leave file unchanged") + break + ;; + "interactiv shell") + echo -e "// edit ${_Red}${dst}${_creset} to your needs" + echo -e "// exit with [${_BCyan}CTRL-D${_creset}]" + sudo -H -u "${owner}" -i + $DIFF_CMD "${dst}" "${template_file}" + echo + echo -e "// ${_BBlack}did you edit file ...${_creset}" + echo -en "// ${_Red}${dst}${_creset}" + if ask_yn "//${_BBlack}... to your needs?${_creset}"; then + break + fi + ;; + "diff files") + $DIFF_CMD "${dst}" "${template_file}" | prefix_stdout + esac + done +} + + +service_is_available() { + + # usage: service_is_available <URL> + + [[ -z $1 ]] && die_caller 42 "missing argument <URL>" + local URL="$1" + http_code=$(curl -H 'Cache-Control: no-cache' \ + --silent -o /dev/null --head --write-out '%{http_code}' --insecure \ + "${URL}") + exit_val=$? + if [[ $exit_val = 0 ]]; then + info_msg "got $http_code from ${URL}" + fi + case "$http_code" in + 404|410|423) exit_val=$http_code;; + esac + return "$exit_val" +} + +# golang +# ------ + +go_is_available() { + + # usage: go_is_available $SERVICE_USER && echo "go is installed!" + + sudo -i -u "${1}" which go &>/dev/null +} + +install_go() { + + # usage: install_go "${GO_PKG_URL}" "${GO_TAR}" "${SERVICE_USER}" + + local _svcpr=" ${_Yellow}|${3}|${_creset} " + + rst_title "Install Go in user's HOME" section + + rst_para "download and install go binary .." + cache_download "${1}" "${2}" + + tee_stderr 0.1 <<EOF | sudo -i -u "${3}" | prefix_stdout "$_svcpr" +echo \$PATH +echo \$GOPATH +mkdir -p \$HOME/local +rm -rf \$HOME/local/go +tar -C \$HOME/local -xzf ${CACHE}/${2} +EOF + sudo -i -u "${3}" <<EOF | prefix_stdout +! which go >/dev/null && echo "ERROR - Go Installation not found in PATH!?!" +which go >/dev/null && go version && echo "congratulations -- Go installation OK :)" +EOF +} + +# system accounts +# --------------- + +service_account_is_available() { + + # usage: service_account_is_available "$SERVICE_USER" && echo "OK" + + sudo -i -u "$1" echo \$HOME &>/dev/null +} + +drop_service_account() { + + # usage: drop_service_account "${SERVICE_USER}" + + rst_title "Drop ${1} HOME" section + if ask_yn "Do you really want to drop ${1} home folder?"; then + userdel -r -f "${1}" 2>&1 | prefix_stdout + else + rst_para "Leave HOME folder $(du -sh "${1}") unchanged." + fi +} + +interactive_shell(){ + + # usage: interactive_shell "${SERVICE_USER}" + + echo -e "// exit with [${_BCyan}CTRL-D${_creset}]" + sudo -H -u "${1}" -i +} + + +# systemd +# ------- + +SYSTEMD_UNITS="${SYSTEMD_UNITS:-/lib/systemd/system}" + +systemd_install_service() { + + # usage: systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" + + rst_title "Install System-D Unit ${1}" section + echo + install_template "${2}" root root 644 + wait_key + systemd_activate_service "${1}" +} + +systemd_remove_service() { + + # usage: systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" + + if ! ask_yn "Do you really want to deinstall systemd unit ${1}?"; then + return 42 + fi + systemd_deactivate_service "${1}" + rm "${2}" 2>&1 | prefix_stdout +} + +systemd_activate_service() { + + # usage: systemd_activate_service "${SERVICE_NAME}" + + rst_title "Activate ${1} (service)" section + echo + tee_stderr <<EOF | bash 2>&1 +systemctl enable ${1}.service +systemctl restart ${1}.service +EOF + tee_stderr <<EOF | bash 2>&1 +systemctl status --no-pager ${1}.service +EOF +} + +systemd_deactivate_service() { + + # usage: systemd_deactivate_service "${SERVICE_NAME}" + + rst_title "De-Activate ${1} (service)" section + echo + tee_stderr <<EOF | bash 2>&1 | prefix_stdout +systemctl stop ${1}.service +systemctl disable ${1}.service +EOF +} + +systemd_restart_service() { + + # usage: systemd_restart_service "${SERVICE_NAME}" + + rst_title "Restart ${1} (service)" section + echo + tee_stderr <<EOF | bash 2>&1 +systemctl restart ${1}.service +EOF + tee_stderr <<EOF | bash 2>&1 +systemctl status --no-pager ${1}.service +EOF +} + + +# nginx +# ----- + +nginx_distro_setup() { + # shellcheck disable=SC2034 + + NGINX_DEFAULT_SERVER=/etc/nginx/nginx.conf + + # Including *location* directives from a dedicated config-folder into the + # server directive is, what what fedora (already) does. + NGINX_APPS_ENABLED="/etc/nginx/default.d" + + # We add a apps-available folder and linking configurations into the + # NGINX_APPS_ENABLED folder. See also nginx_include_apps_enabled(). + NGINX_APPS_AVAILABLE="/etc/nginx/default.apps-available" + + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) + NGINX_PACKAGES="nginx" + NGINX_DEFAULT_SERVER=/etc/nginx/sites-available/default + ;; + arch-*) + NGINX_PACKAGES="nginx-mainline" + ;; + fedora-*) + NGINX_PACKAGES="nginx" + ;; + *) + err_msg "$DIST_ID-$DIST_VERS: nginx not yet implemented" + ;; + esac +} +nginx_distro_setup + +install_nginx(){ + info_msg "installing nginx ..." + pkg_install "${NGINX_PACKAGES}" + case $DIST_ID-$DIST_VERS in + arch-*|fedora-*) + systemctl enable nginx + systemctl start nginx + ;; + esac +} + +nginx_is_installed() { + command -v nginx &>/dev/null +} + +nginx_reload() { + + info_msg "reload nginx .." + echo + if ! nginx -t; then + err_msg "testing nginx configuration failed" + return 42 + fi + systemctl restart nginx +} + +nginx_install_app() { + + # usage: nginx_install_app [<template option> ...] <myapp> + # + # <template option>: see install_template + + local template_opts=() + local pos_args=("$0") + + for i in "$@"; do + case $i in + -*) template_opts+=("$i");; + *) pos_args+=("$i");; + esac + done + + nginx_include_apps_enabled "${NGINX_DEFAULT_SERVER}" + + install_template "${template_opts[@]}" \ + "${NGINX_APPS_AVAILABLE}/${pos_args[1]}" \ + root root 644 + nginx_enable_app "${pos_args[1]}" + info_msg "installed nginx app: ${pos_args[1]}" +} + +nginx_include_apps_enabled() { + + # Add the *NGINX_APPS_ENABLED* infrastruture to a nginx server block. Such + # infrastruture is already known from fedora, including apps (location + # directives) from the /etc/nginx/default.d folder into the *default* nginx + # server. + + # usage: nginx_include_apps_enabled <config-file> + # + # config-file: Config file with server directive in. + + [[ -z $1 ]] && die_caller 42 "missing argument <config-file>" + local server_conf="$1" + + # include /etc/nginx/default.d/*.conf; + local include_directive="include ${NGINX_APPS_ENABLED}/*.conf;" + local include_directive_re="^\s*include ${NGINX_APPS_ENABLED}/\*\.conf;" + + info_msg "checking existence: '${include_directive}' in file ${server_conf}" + if grep "${include_directive_re}" "${server_conf}"; then + info_msg "OK, already exists." + return + fi + + info_msg "add missing directive: '${include_directive}'" + cp "${server_conf}" "${server_conf}.bak" + + ( + local line + local stage=0 + while IFS= read -r line + do + echo "$line" + if [[ $stage = 0 ]]; then + if [[ $line =~ ^[[:space:]]*server*[[:space:]]*\{ ]]; then + stage=1 + fi + fi + + if [[ $stage = 1 ]]; then + echo " # Load configuration files for the default server block." + echo " $include_directive" + echo "" + stage=2 + fi + done < "${server_conf}.bak" + ) > "${server_conf}" + +} + +nginx_remove_app() { + + # usage: nginx_remove_app <myapp.conf> + + info_msg "remove nginx app: $1" + nginx_dissable_app "$1" + rm -f "${NGINX_APPS_AVAILABLE}/$1" +} + +nginx_enable_app() { + + # usage: nginx_enable_app <myapp.conf> + + local CONF="$1" + + info_msg "enable nginx app: ${CONF}" + mkdir -p "${NGINX_APPS_ENABLED}" + rm -f "${NGINX_APPS_ENABLED}/${CONF}" + ln -s "${NGINX_APPS_AVAILABLE}/${CONF}" "${NGINX_APPS_ENABLED}/${CONF}" + nginx_reload +} + +nginx_dissable_app() { + + # usage: nginx_disable_app <myapp.conf> + + local CONF="$1" + + info_msg "disable nginx app: ${CONF}" + rm -f "${NGINX_APPS_ENABLED}/${CONF}" + nginx_reload +} + + +# Apache +# ------ + +apache_distro_setup() { + # shellcheck disable=SC2034 + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) + # debian uses the /etc/apache2 path, while other distros use + # the apache default at /etc/httpd + APACHE_SITES_AVAILABLE="/etc/apache2/sites-available" + APACHE_SITES_ENABLED="/etc/apache2/sites-enabled" + APACHE_MODULES="/usr/lib/apache2/modules" + APACHE_PACKAGES="apache2" + ;; + arch-*) + APACHE_SITES_AVAILABLE="/etc/httpd/sites-available" + APACHE_SITES_ENABLED="/etc/httpd/sites-enabled" + APACHE_MODULES="modules" + APACHE_PACKAGES="apache" + ;; + fedora-*) + APACHE_SITES_AVAILABLE="/etc/httpd/sites-available" + APACHE_SITES_ENABLED="/etc/httpd/sites-enabled" + APACHE_MODULES="modules" + APACHE_PACKAGES="httpd" + ;; + *) + err_msg "$DIST_ID-$DIST_VERS: apache not yet implemented" + ;; + esac +} + +apache_distro_setup + +install_apache(){ + info_msg "installing apache ..." + pkg_install "$APACHE_PACKAGES" + case $DIST_ID-$DIST_VERS in + arch-*|fedora-*) + if ! grep "IncludeOptional sites-enabled" "/etc/httpd/conf/httpd.conf"; then + echo "IncludeOptional sites-enabled/*.conf" >> "/etc/httpd/conf/httpd.conf" + fi + systemctl enable httpd + systemctl start httpd + ;; + esac +} + +apache_is_installed() { + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) (command -v apachectl) &>/dev/null;; + arch-*) (command -v httpd) &>/dev/null;; + fedora-*) (command -v httpd) &>/dev/null;; + esac +} + +apache_reload() { + + info_msg "reload apache .." + echo + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) + sudo -H apachectl configtest + sudo -H systemctl force-reload apache2 + ;; + arch-*| fedora-*) + sudo -H httpd -t + sudo -H systemctl force-reload httpd + ;; + esac +} + +apache_install_site() { + + # usage: apache_install_site [<template option> ...] <mysite.conf> + # + # <template option>: see install_template + + local template_opts=() + local pos_args=("$0") + + for i in "$@"; do + case $i in + -*) template_opts+=("$i");; + *) pos_args+=("$i");; + esac + done + + install_template "${template_opts[@]}" \ + "${APACHE_SITES_AVAILABLE}/${pos_args[1]}" \ + root root 644 + apache_enable_site "${pos_args[1]}" + info_msg "installed apache site: ${pos_args[1]}" +} + +apache_remove_site() { + + # usage: apache_remove_site <mysite.conf> + + info_msg "remove apache site: $1" + apache_dissable_site "$1" + rm -f "${APACHE_SITES_AVAILABLE}/$1" +} + +apache_enable_site() { + + # usage: apache_enable_site <mysite.conf> + + local CONF="$1" + + info_msg "enable apache site: ${CONF}" + + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) + sudo -H a2ensite -q "${CONF}" + ;; + arch-*) + mkdir -p "${APACHE_SITES_ENABLED}" + rm -f "${APACHE_SITES_ENABLED}/${CONF}" + ln -s "${APACHE_SITES_AVAILABLE}/${CONF}" "${APACHE_SITES_ENABLED}/${CONF}" + ;; + fedora-*) + mkdir -p "${APACHE_SITES_ENABLED}" + rm -f "${APACHE_SITES_ENABLED}/${CONF}" + ln -s "${APACHE_SITES_AVAILABLE}/${CONF}" "${APACHE_SITES_ENABLED}/${CONF}" + ;; + esac + apache_reload +} + +apache_dissable_site() { + + # usage: apache_disable_site <mysite.conf> + + local CONF="$1" + + info_msg "disable apache site: ${CONF}" + + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) + sudo -H a2dissite -q "${CONF}" + ;; + arch-*) + rm -f "${APACHE_SITES_ENABLED}/${CONF}" + ;; + fedora-*) + rm -f "${APACHE_SITES_ENABLED}/${CONF}" + ;; + esac + apache_reload +} + +# uWSGI +# ----- + +uWSGI_SETUP="${uWSGI_SETUP:=/etc/uwsgi}" +uWSGI_USER= +uWSGI_GROUP= + +# How distros manage uWSGI apps is very different. From uWSGI POV read: +# - https://uwsgi-docs.readthedocs.io/en/latest/Management.html + +uWSGI_distro_setup() { + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) + # init.d --> /usr/share/doc/uwsgi/README.Debian.gz + # For uWSGI debian uses the LSB init process, this might be changed + # one day, see https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=833067 + uWSGI_APPS_AVAILABLE="${uWSGI_SETUP}/apps-available" + uWSGI_APPS_ENABLED="${uWSGI_SETUP}/apps-enabled" + uWSGI_PACKAGES="uwsgi" + ;; + arch-*) + # systemd --> /usr/lib/systemd/system/uwsgi@.service + # For uWSGI archlinux uses systemd template units, see + # - http://0pointer.de/blog/projects/instances.html + # - https://uwsgi-docs.readthedocs.io/en/latest/Systemd.html#one-service-per-app-in-systemd + uWSGI_APPS_AVAILABLE="${uWSGI_SETUP}/apps-archlinux" + uWSGI_APPS_ENABLED="${uWSGI_SETUP}" + uWSGI_PACKAGES="uwsgi" + ;; + fedora-*) + # systemd --> /usr/lib/systemd/system/uwsgi.service + # The unit file starts uWSGI in emperor mode (/etc/uwsgi.ini), see + # - https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html + uWSGI_APPS_AVAILABLE="${uWSGI_SETUP}/apps-available" + uWSGI_APPS_ENABLED="${uWSGI_SETUP}.d" + uWSGI_PACKAGES="uwsgi" + uWSGI_USER="uwsgi" + uWSGI_GROUP="uwsgi" + ;; + *) + err_msg "$DIST_ID-$DIST_VERS: uWSGI not yet implemented" + ;; +esac +} + +uWSGI_distro_setup + +install_uwsgi(){ + info_msg "installing uwsgi ..." + pkg_install "$uWSGI_PACKAGES" + case $DIST_ID-$DIST_VERS in + fedora-*) + # enable & start should be called once at uWSGI installation time + systemctl enable uwsgi + systemctl restart uwsgi + ;; + esac +} + +uWSGI_restart() { + + # usage: uWSGI_restart() <myapp.ini> + + local CONF="$1" + + [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>" + info_msg "restart uWSGI service" + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) + # the 'service' method seems broken in that way, that it (re-)starts + # the whole uwsgi process. + service uwsgi restart "${CONF%.*}" + ;; + arch-*) + # restart systemd template instance + if uWSGI_app_available "${CONF}"; then + systemctl restart "uwsgi@${CONF%.*}" + else + info_msg "[uWSGI:systemd-template] ${CONF} not installed (no need to restart)" + fi + ;; + fedora-*) + # in emperor mode, just touch the file to restart + if uWSGI_app_enabled "${CONF}"; then + touch "${uWSGI_APPS_ENABLED}/${CONF}" + # it seems, there is a polling time in between touch and restart + # of the service. + sleep 3 + else + info_msg "[uWSGI:emperor] ${CONF} not installed (no need to restart)" + fi + ;; + *) + err_msg "$DIST_ID-$DIST_VERS: uWSGI not yet implemented" + return 42 + ;; + esac +} + +uWSGI_prepare_app() { + + # usage: uWSGI_prepare_app <myapp.ini> + + [[ -z $1 ]] && die_caller 42 "missing argument <myapp.ini>" + + local APP="${1%.*}" + + case $DIST_ID-$DIST_VERS in + fedora-*) + # in emperor mode, the uwsgi user is the owner of the sockets + info_msg "prepare (uwsgi:uwsgi) /run/uwsgi/app/${APP}" + mkdir -p "/run/uwsgi/app/${APP}" + chown -R "uwsgi:uwsgi" "/run/uwsgi/app/${APP}" + ;; + *) + info_msg "prepare (${SERVICE_USER}:${SERVICE_GROUP}) /run/uwsgi/app/${APP}" + mkdir -p "/run/uwsgi/app/${APP}" + chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "/run/uwsgi/app/${APP}" + ;; + esac +} + + +uWSGI_app_available() { + # usage: uWSGI_app_available <myapp.ini> + local CONF="$1" + + [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>" + [[ -f "${uWSGI_APPS_AVAILABLE}/${CONF}" ]] +} + +uWSGI_install_app() { + + # usage: uWSGI_install_app [<template option> ...] <myapp.ini> + # + # <template option>: see install_template + + local pos_args=("$0") + + for i in "$@"; do + case $i in + -*) template_opts+=("$i");; + *) pos_args+=("$i");; + esac + done + uWSGI_prepare_app "${pos_args[1]}" + mkdir -p "${uWSGI_APPS_AVAILABLE}" + install_template "${template_opts[@]}" \ + "${uWSGI_APPS_AVAILABLE}/${pos_args[1]}" \ + root root 644 + uWSGI_enable_app "${pos_args[1]}" + uWSGI_restart "${pos_args[1]}" + info_msg "uWSGI app: ${pos_args[1]} is installed" +} + +uWSGI_remove_app() { + + # usage: uWSGI_remove_app <myapp.ini> + + local CONF="$1" + + [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>" + info_msg "remove uWSGI app: ${CONF}" + uWSGI_disable_app "${CONF}" + uWSGI_restart "${CONF}" + rm -f "${uWSGI_APPS_AVAILABLE}/${CONF}" +} + +uWSGI_app_enabled() { + # usage: uWSGI_app_enabled <myapp.ini> + + local exit_val=0 + local CONF="$1" + + [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>" + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) + [[ -f "${uWSGI_APPS_ENABLED}/${CONF}" ]] + exit_val=$? + ;; + arch-*) + systemctl -q is-enabled "uwsgi@${CONF%.*}" + exit_val=$? + ;; + fedora-*) + [[ -f "${uWSGI_APPS_ENABLED}/${CONF}" ]] + exit_val=$? + ;; + *) + # FIXME + err_msg "$DIST_ID-$DIST_VERS: uWSGI not yet implemented" + exit_val=1 + ;; + esac + return $exit_val +} + +# shellcheck disable=SC2164 +uWSGI_enable_app() { + + # usage: uWSGI_enable_app <myapp.ini> + + local CONF="$1" + + [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>" + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) + mkdir -p "${uWSGI_APPS_ENABLED}" + rm -f "${uWSGI_APPS_ENABLED}/${CONF}" + ln -s "${uWSGI_APPS_AVAILABLE}/${CONF}" "${uWSGI_APPS_ENABLED}/${CONF}" + info_msg "enabled uWSGI app: ${CONF} (restart required)" + ;; + arch-*) + mkdir -p "${uWSGI_APPS_ENABLED}" + rm -f "${uWSGI_APPS_ENABLED}/${CONF}" + ln -s "${uWSGI_APPS_AVAILABLE}/${CONF}" "${uWSGI_APPS_ENABLED}/${CONF}" + systemctl enable "uwsgi@${CONF%.*}" + info_msg "enabled uWSGI app: ${CONF} (restart required)" + ;; + fedora-*) + mkdir -p "${uWSGI_APPS_ENABLED}" + rm -f "${uWSGI_APPS_ENABLED}/${CONF}" + ln -s "${uWSGI_APPS_AVAILABLE}/${CONF}" "${uWSGI_APPS_ENABLED}/${CONF}" + chown "${uWSGI_USER}:${uWSGI_GROUP}" "${uWSGI_APPS_ENABLED}/${CONF}" + info_msg "enabled uWSGI app: ${CONF}" + ;; + *) + # FIXME + err_msg "$DIST_ID-$DIST_VERS: uWSGI not yet implemented" + ;; + esac +} + +uWSGI_disable_app() { + + # usage: uWSGI_disable_app <myapp.ini> + + local CONF="$1" + + [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>" + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) + service uwsgi stop "${CONF%.*}" + rm -f "${uWSGI_APPS_ENABLED}/${CONF}" + info_msg "disabled uWSGI app: ${CONF} (restart uWSGI required)" + ;; + arch-*) + systemctl stop "uwsgi@${CONF%.*}" + systemctl disable "uwsgi@${CONF%.*}" + rm -f "${uWSGI_APPS_ENABLED}/${CONF}" + ;; + fedora-*) + # in emperor mode, just remove the app.ini file + rm -f "${uWSGI_APPS_ENABLED}/${CONF}" + ;; + *) + # FIXME + err_msg "$DIST_ID-$DIST_VERS: uWSGI not yet implemented" + ;; + esac +} + +# distro's package manager +# ------------------------ + +_apt_pkg_info_is_updated=0 + +pkg_install() { + + # usage: TITEL='install foobar' pkg_install foopkg barpkg + + rst_title "${TITLE:-installation of packages}" section + echo -e "\npackage(s)::\n" + # shellcheck disable=SC2068 + echo " " $@ | $FMT + + if ! ask_yn "Should packages be installed?" Yn 30; then + return 42 + fi + case $DIST_ID in + ubuntu|debian) + if [[ $_apt_pkg_info_is_updated == 0 ]]; then + export _apt_pkg_info_is_updated=1 + apt update + fi + # shellcheck disable=SC2068 + apt-get install -m -y $@ + ;; + arch) + # shellcheck disable=SC2068 + pacman -Sy --noconfirm $@ + ;; + fedora) + # shellcheck disable=SC2068 + dnf install -y $@ + ;; + esac +} + +pkg_remove() { + + # usage: TITEL='remove foobar' pkg_remove foopkg barpkg + + rst_title "${TITLE:-remove packages}" section + echo -e "\npackage(s)::\n" + # shellcheck disable=SC2068 + echo " " $@ | $FMT + + if ! ask_yn "Should packages be removed (purge)?" Yn 30; then + return 42 + fi + case $DIST_ID in + ubuntu|debian) + # shellcheck disable=SC2068 + apt-get purge --autoremove --ignore-missing -y $@ + ;; + arch) + # shellcheck disable=SC2068 + pacman -R --noconfirm $@ + ;; + fedora) + # shellcheck disable=SC2068 + dnf remove -y $@ + ;; + esac +} + +pkg_is_installed() { + + # usage: pkg_is_install foopkg || pkg_install foopkg + + case $DIST_ID in + ubuntu|debian) + dpkg -l "$1" &> /dev/null + return $? + ;; + arch) + pacman -Qsq "$1" &> /dev/null + return $? + ;; + fedora) + dnf list -q --installed "$1" &> /dev/null + return $? + ;; + esac +} + +# git tooling +# ----------- + +# shellcheck disable=SC2164 +git_clone() { + + # usage: + # + # git_clone <url> <name> [<branch> [<user>]] + # git_clone <url> <path> [<branch> [<user>]] + # + # First form uses $CACHE/<name> as destination folder, second form clones + # into <path>. If repository is allready cloned, pull from <branch> and + # update working tree (if needed, the caller has to stash local changes). + # + # git clone https://github.com/asciimoo/searx searx-src origin/master searxlogin + # + + local url="$1" + local dest="$2" + local branch="$3" + local user="$4" + local bash_cmd="bash" + local remote="origin" + + if [[ ! "${dest:0:1}" = "/" ]]; then + dest="$CACHE/$dest" + fi + + [[ -z $branch ]] && branch=master + [[ -z $user ]] && [[ -n "${SUDO_USER}" ]] && user="${SUDO_USER}" + [[ -n $user ]] && bash_cmd="sudo -H -u $user -i" + + if [[ -d "${dest}" ]] ; then + info_msg "already cloned: $dest" + tee_stderr 0.1 <<EOF | $bash_cmd 2>&1 | prefix_stdout " ${_Yellow}|$user|${_creset} " +cd "${dest}" +git checkout -m -B "$branch" --track "$remote/$branch" +git pull --all +EOF + else + info_msg "clone into: $dest" + tee_stderr 0.1 <<EOF | $bash_cmd 2>&1 | prefix_stdout " ${_Yellow}|$user|${_creset} " +mkdir -p "$(dirname "$dest")" +cd "$(dirname "$dest")" +git clone --branch "$branch" --origin "$remote" "$url" "$(basename "$dest")" +EOF + fi +} + +# containers +# ---------- + +in_container() { + # Test if shell runs in a container. + # + # usage: in_container && echo "process running inside a LXC container" + # in_container || echo "process is not running inside a LXC container" + # + # sudo_or_exit + # hint: Reads init process environment, therefore root access is required! + # to be safe, take a look at the environment of process 1 (/sbin/init) + # grep -qa 'container=lxc' /proc/1/environ + + # see lxc_init_container_env + [[ -f /.lxcenv ]] +} + +LXC_ENV_FOLDER= +if in_container; then + # shellcheck disable=SC2034 + LXC_ENV_FOLDER="lxc/$(hostname)/" +fi + +lxc_init_container_env() { + + # usage: lxc_init_container_env <name> + + # Create a /.lxcenv file in the root folder. Call this once after the + # container is inital started and before installing any boilerplate stuff. + + info_msg "create /.lxcenv in container $1" + cat <<EOF | lxc exec "${1}" -- bash | prefix_stdout "[${_BBlue}${1}${_creset}] " +touch "/.lxcenv" +ls -l "/.lxcenv" +EOF +} + +# apt packages +LXC_BASE_PACKAGES_debian="bash git build-essential python3 virtualenv" + +# pacman packages +LXC_BASE_PACKAGES_arch="bash git base-devel python python-virtualenv" + +# dnf packages +LXC_BASE_PACKAGES_fedora="bash git @development-tools python virtualenv" + +case $DIST_ID in + ubuntu|debian) LXC_BASE_PACKAGES="${LXC_BASE_PACKAGES_debian}" ;; + arch) LXC_BASE_PACKAGES="${LXC_BASE_PACKAGES_arch}" ;; + fedora) LXC_BASE_PACKAGES="${LXC_BASE_PACKAGES_fedora}" ;; + *) err_msg "$DIST_ID-$DIST_VERS: pkg_install LXC_BASE_PACKAGES not yet implemented" ;; +esac + +lxc_install_base_packages() { + info_msg "install LXC_BASE_PACKAGES in container $1" + pkg_install "${LXC_BASE_PACKAGES}" +} + + +lxc_image_copy() { + + # usage: lxc_copy_image <remote image> <local image> + # + # lxc_copy_image "images:ubuntu/19.10" "ubu1910" + + if lxc_image_exists "local:${LXC_SUITE[i+1]}"; then + info_msg "image ${LXC_SUITE[i]} already copied --> ${LXC_SUITE[i+1]}" + else + info_msg "copy image locally ${LXC_SUITE[i]} --> ${LXC_SUITE[i+1]}" + lxc image copy "${LXC_SUITE[i]}" local: \ + --alias "${LXC_SUITE[i+1]}" | prefix_stdout + fi +} + +lxc_init_container() { + + # usage: lxc_init_container <image name> <container name> + + local image_name="$1" + local container_name="$2" + + if lxc info "${container_name}" &>/dev/null; then + info_msg "container '${container_name}' already exists" + else + info_msg "create container instance: ${container_name}" + lxc init "local:${image_name}" "${container_name}" + fi +} + +lxc_exists(){ + + # usage: lxc_exists <name> || echo "container <name> does not exists" + + lxc info "$1" &>/dev/null +} + +lxc_image_exists(){ + # usage: lxc_image_exists <alias> || echo "image <alias> does locally not exists" + + lxc image info "local:$1" &>/dev/null + +} + +lxc_delete_container() { + + # usage: lxc_delete_container <container-name> + + if lxc info "$1" &>/dev/null; then + info_msg "stop & delete instance ${_BBlue}${1}${_creset}" + lxc stop "$1" &>/dev/null + lxc delete "$1" | prefix_stdout + else + warn_msg "instance '$1' does not exist / can't delete :o" + fi +} + +lxc_delete_local_image() { + + # usage: lxc_delete_local_image <container-name> + + info_msg "delete image 'local:$i'" + lxc image delete "local:$i" +} + + +# IP +# -- + +global_IPs(){ + # usage: global_IPS + # + # print list of host's SCOPE global addresses and adapters e.g:: + # + # $ global_IPs + # enp4s0|192.168.1.127 + # lxdbr0|10.246.86.1 + # lxdbr0|fd42:8c58:2cd:b73f::1 + + ip -o addr show | sed -nr 's/[0-9]*:\s*([a-z0-9]*).*inet[6]?\s*([a-z0-9.:]*).*scope global.*/\1|\2/p' +} + +primary_ip() { + + case $DIST_ID in + arch) + ip -o addr show \ + | sed -nr 's/[0-9]*:\s*([a-z0-9]*).*inet[6]?\s*([a-z0-9.:]*).*scope global.*/\2/p' \ + | head -n 1 + ;; + *) hostname -I | cut -d' ' -f1 ;; + esac +} + +# URL +# --- + +url_replace_hostname(){ + + # usage: url_replace_hostname <url> <new hostname> + + # to replace hostname by primary IP:: + # + # url_replace_hostname http://searx-ubu1604/morty $(primary_ip) + # http://10.246.86.250/morty + + # shellcheck disable=SC2001 + echo "$1" | sed "s|\(http[s]*://\)[^/]*\(.*\)|\1$2\2|" +} diff --git a/utils/lxc-searx.env b/utils/lxc-searx.env new file mode 100644 index 000000000..a51312fb8 --- /dev/null +++ b/utils/lxc-searx.env @@ -0,0 +1,95 @@ +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- +# SPDX-License-Identifier: AGPL-3.0-or-later +# shellcheck shell=bash + +# This file is a setup of a LXC suite. It is sourced from different context, do +# not manipulate the environment directly, implement functions and manipulate +# environment only is subshells! + +# ---------------------------------------------------------------------------- +# config +# ---------------------------------------------------------------------------- + +# shellcheck disable=SC2034 +LXC_SUITE_NAME="searx" +lxc_set_suite_env() { + # name of https://images.linuxcontainers.org + export LINUXCONTAINERS_ORG_NAME="${LINUXCONTAINERS_ORG_NAME:-images}" + export LXC_HOST_PREFIX="${LXC_SUITE_NAME:-searx}" + export LXC_SUITE=( + + # to disable containers, comment out lines .. + + # end of standard support see https://wiki.ubuntu.com/Releases + "$LINUXCONTAINERS_ORG_NAME:ubuntu/16.04" "ubu1604" # April 2021 + "$LINUXCONTAINERS_ORG_NAME:ubuntu/18.04" "ubu1804" # April 2023 + "$LINUXCONTAINERS_ORG_NAME:ubuntu/19.10" "ubu1910" # July 2020 + "$LINUXCONTAINERS_ORG_NAME:ubuntu/20.04" "ubu2004" # future (EOL 2030) + + # EOL see https://fedoraproject.org/wiki/Releases + "$LINUXCONTAINERS_ORG_NAME:fedora/31" "fedora31" + + # rolling releases see https://www.archlinux.org/releng/releases/ + "$LINUXCONTAINERS_ORG_NAME:archlinux" "archlinux" + ) + + PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}" + if in_container; then + # container hostnames do not have a DNS entry: use primary IP! + PUBLIC_URL="http://$(primary_ip)/searx" + + # make GUEST's services public to the HOST + FILTRON_API="0.0.0.0:4005" + FILTRON_LISTEN="0.0.0.0:4004" + MORTY_LISTEN="0.0.0.0:3000" + + # export LXC specific environment + export PUBLIC_URL FILTRON_API FILTRON_LISTEN MORTY_LISTEN + fi +} + +lxc_suite_install_info() { + ( + lxc_set_suite_env + cat <<EOF +LXC suite: ${LXC_SUITE_NAME} --> ${PUBLIC_URL} + suite includes searx, morty & filtron +suite images: +$(echo " ${LOCAL_IMAGES[*]}" | $FMT) +suite containers: +$(echo " ${CONTAINERS[*]}" | $FMT) +EOF + ) + } + +lxc_suite_install() { + ( + lxc_set_suite_env + FORCE_TIMEOUT=0 + export FORCE_TIMEOUT + "${LXC_REPO_ROOT}/utils/searx.sh" install all + "${LXC_REPO_ROOT}/utils/morty.sh" install all + "${LXC_REPO_ROOT}/utils/filtron.sh" install all + + rst_title "suite installation finished ($(hostname))" part + lxc_suite_info + echo + ) +} + +lxc_suite_info() { + ( + lxc_set_suite_env + for ip in $(global_IPs) ; do + if [[ $ip =~ .*:.* ]]; then + info_msg "(${ip%|*}) IPv6: http://[${ip#*|}]" + else + # IPv4: + # shellcheck disable=SC2034,SC2031 + info_msg "(${ip%|*}) filtron: http://${ip#*|}:4004/ $PUBLIC_URL" + info_msg "(${ip%|*}) morty: http://${ip#*|}:3000/ $PUBLIC_URL_MORTY" + info_msg "(${ip%|*}) docs-live: http://${ip#*|}:8080/" + fi + done + ) +} diff --git a/utils/lxc.sh b/utils/lxc.sh new file mode 100755 index 000000000..a0688bc07 --- /dev/null +++ b/utils/lxc.sh @@ -0,0 +1,552 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- +# SPDX-License-Identifier: AGPL-3.0-or-later + +# shellcheck source=utils/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +source_dot_config + +# load environment of the LXC suite +LXC_ENV="${LXC_ENV:-${REPO_ROOT}/utils/lxc-searx.env}" +source "$LXC_ENV" +lxc_set_suite_env + +# ---------------------------------------------------------------------------- +# config +# ---------------------------------------------------------------------------- +# +# read also: +# - https://lxd.readthedocs.io/en/latest/ + +LXC_HOST_PREFIX="${LXC_HOST_PREFIX:-test}" + +# where all folders from HOST are mounted +LXC_SHARE_FOLDER="/share" +LXC_REPO_ROOT="${LXC_SHARE_FOLDER}/$(basename "${REPO_ROOT}")" + +ubu1604_boilerplate=" +export DEBIAN_FRONTEND=noninteractive +apt-get update -y +apt-get upgrade -y +apt-get install -y git curl wget +" +ubu1804_boilerplate="$ubu1604_boilerplate" +ubu1904_boilerplate="$ubu1804_boilerplate" +ubu1910_boilerplate="$ubu1904_boilerplate" + +# shellcheck disable=SC2034 +ubu2004_boilerplate=" +$ubu1910_boilerplate +echo 'Set disable_coredump false' >> /etc/sudo.conf +" + +# shellcheck disable=SC2034 +archlinux_boilerplate=" +pacman -Syu --noconfirm +pacman -S --noconfirm inetutils git curl wget sudo +echo 'Set disable_coredump false' >> /etc/sudo.conf +" + +# shellcheck disable=SC2034 +fedora31_boilerplate=" +dnf update -y +dnf install -y git curl wget hostname +echo 'Set disable_coredump false' >> /etc/sudo.conf +" + +REMOTE_IMAGES=() +CONTAINERS=() +LOCAL_IMAGES=() + +for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do + REMOTE_IMAGES=("${REMOTE_IMAGES[@]}" "${LXC_SUITE[i]}") + CONTAINERS=("${CONTAINERS[@]}" "${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}") + LOCAL_IMAGES=("${LOCAL_IMAGES[@]}" "${LXC_SUITE[i+1]}") +done + +HOST_USER="${SUDO_USER:-$USER}" +HOST_USER_ID=$(id -u "${HOST_USER}") +HOST_GROUP_ID=$(id -g "${HOST_USER}") + +# ---------------------------------------------------------------------------- +usage() { +# ---------------------------------------------------------------------------- + _cmd="$(basename "$0")" + cat <<EOF +usage:: + $_cmd build [containers|<name>] + $_cmd copy [images] + $_cmd remove [containers|<name>|images] + $_cmd [start|stop] [containers|<name>] + $_cmd show [images|suite|info|config [<name>]] + $_cmd cmd [--|<name>] '...' + $_cmd install [suite|base [<name>]] + +build + :containers: build, launch all containers and 'install base' packages + :<name>: build, launch container <name> and 'install base' packages +copy: + :images: copy remote images of the suite into local storage +remove + :containers: delete all 'containers' or only <container-name> + :images: delete local images of the suite +start/stop + :containers: start/stop all 'containers' from the suite + :<name>: start/stop container <name> from suite +show + :info: show info of all (or <name>) containers from LXC suite + :config: show config of all (or <name>) containers from the LXC suite + :suite: show services of all (or <name>) containers from the LXC suite + :images: show information of local images +cmd + use single qoutes to evaluate in container's bash, e.g.: 'echo \$(hostname)' + -- run command '...' in all containers of the LXC suite + :<name>: run command '...' in container <name> +install + :base: prepare LXC; install basic packages + :suite: install LXC ${LXC_SUITE_NAME} suite into all (or <name>) containers + +EOF + usage_containers + [ -n "${1+x}" ] && err_msg "$1" +} + +usage_containers() { + lxc_suite_install_info + [ -n "${1+x}" ] && err_msg "$1" +} + +lxd_info() { + + cat <<EOF + +LXD is needed, to install run:: + + snap install lxd + lxd init --auto + +EOF +} + +main() { + + local exit_val + local _usage="unknown or missing $1 command $2" + + # don't check prerequisite when in recursion + if [[ ! $1 == __* ]]; then + if ! in_container; then + ! required_commands lxc && lxd_info && exit 42 + fi + [[ -z $LXC_SUITE ]] && err_msg "missing LXC_SUITE" && exit 42 + fi + + case $1 in + --getenv) var="$2"; echo "${!var}"; exit 0;; + -h|--help) usage; exit 0;; + + build) + sudo_or_exit + case $2 in + ${LXC_HOST_PREFIX}-*) build_container "$2" ;; + ''|--|containers) build_all_containers ;; + *) usage "$_usage"; exit 42;; + esac + ;; + copy) + case $2 in + ''|images) lxc_copy_images_localy;; + *) usage "$_usage"; exit 42;; + esac + ;; + remove) + sudo_or_exit + case $2 in + ''|--|containers) remove_containers ;; + images) lxc_delete_images_localy ;; + ${LXC_HOST_PREFIX}-*) + ! lxc_exists "$2" && warn_msg "container not yet exists: $2" && exit 0 + if ask_yn "Do you really want to delete container $2"; then + lxc_delete_container "$2" + fi + ;; + *) usage "uknown or missing container <name> $2"; exit 42;; + esac + ;; + start|stop) + sudo_or_exit + case $2 in + ''|--|containers) lxc_cmd "$1" ;; + ${LXC_HOST_PREFIX}-*) + ! lxc_exists "$2" && usage_containers "unknown container: $2" && exit 42 + info_msg "lxc $1 $2" + lxc "$1" "$2" | prefix_stdout "[${_BBlue}${i}${_creset}] " + ;; + *) usage "uknown or missing container <name> $2"; exit 42;; + esac + ;; + show) + sudo_or_exit + case $2 in + suite) + case $3 in + ${LXC_HOST_PREFIX}-*) + lxc exec -t "$3" -- "${LXC_REPO_ROOT}/utils/lxc.sh" __show suite \ + | prefix_stdout "[${_BBlue}$3${_creset}] " + ;; + *) show_suite;; + esac + ;; + images) show_images ;; + config) + case $3 in + ${LXC_HOST_PREFIX}-*) + ! lxc_exists "$3" && usage_containers "unknown container: $3" && exit 42 + lxc config show "$3" | prefix_stdout "[${_BBlue}${3}${_creset}] " + ;; + *) + rst_title "container configurations" + echo + lxc list "$LXC_HOST_PREFIX-" + echo + lxc_cmd config show + ;; + esac + ;; + info) + case $3 in + ${LXC_HOST_PREFIX}-*) + ! lxc_exists "$3" && usage_containers "unknown container: $3" && exit 42 + lxc info "$3" | prefix_stdout "[${_BBlue}${3}${_creset}] " + ;; + *) + rst_title "container info" + echo + lxc_cmd info + ;; + esac + ;; + *) usage "$_usage"; exit 42;; + esac + ;; + __show) + # wrapped show commands, called once in each container + case $2 in + suite) lxc_suite_info ;; + esac + ;; + cmd) + sudo_or_exit + shift + case $1 in + --) shift; lxc_exec "$@" ;; + ${LXC_HOST_PREFIX}-*) + ! lxc_exists "$1" && usage_containers "unknown container: $1" && exit 42 + local name=$1 + shift + lxc_exec_cmd "${name}" "$@" + ;; + *) usage_containers "unknown container: $1" && exit 42 + esac + ;; + install) + sudo_or_exit + case $2 in + suite|base) + case $3 in + ${LXC_HOST_PREFIX}-*) + ! lxc_exists "$3" && usage_containers "unknown container: $3" && exit 42 + lxc_exec_cmd "$3" "${LXC_REPO_ROOT}/utils/lxc.sh" __install "$2" + ;; + ''|--) lxc_exec "${LXC_REPO_ROOT}/utils/lxc.sh" __install "$2" ;; + *) usage_containers "unknown container: $3" && exit 42 + esac + ;; + *) usage "$_usage"; exit 42 ;; + esac + ;; + __install) + # wrapped install commands, called once in each container + # shellcheck disable=SC2119 + case $2 in + suite) lxc_suite_install ;; + base) FORCE_TIMEOUT=0 lxc_install_base_packages ;; + esac + ;; + doc) + echo + echo ".. generic utils/lxc.sh documentation" + ;; + -*) usage "unknown option $1"; exit 42;; + *) usage "unknown or missing command $1"; exit 42;; + esac +} + + +build_all_containers() { + rst_title "Build all LXC containers of suite" + echo + usage_containers + lxc_copy_images_localy + lxc_init_all_containers + lxc_config_all_containers + lxc_boilerplate_all_containers + rst_title "install LXC base packages" section + echo + lxc_exec "${LXC_REPO_ROOT}/utils/lxc.sh" __install base + echo + lxc list "$LXC_HOST_PREFIX" +} + +build_container() { + rst_title "Build container $1" + + local remote_image + local container + local image + local boilerplate_script + + for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do + if [ "${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}" = "$1" ]; then + remote_image="${LXC_SUITE[i]}" + container="${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}" + image="${LXC_SUITE[i+1]}" + boilerplate_script="${image}_boilerplate" + boilerplate_script="${!boilerplate_script}" + break + fi + done + echo + if [ -z "$container" ]; then + err_msg "container $1 unknown" + usage_containers + return 42 + fi + lxc_image_copy "${remote_image}" "${image}" + rst_title "init container" section + lxc_init_container "${image}" "${container}" + rst_title "configure container" section + lxc_config_container "${container}" + rst_title "run LXC boilerplate scripts" section + lxc_install_boilerplate "${container}" "$boilerplate_script" + echo + rst_title "install LXC base packages" section + lxc_exec_cmd "${container}" "${LXC_REPO_ROOT}/utils/lxc.sh" __install base \ + | prefix_stdout "[${_BBlue}${container}${_creset}] " + echo + lxc list "$container" +} + +remove_containers() { + rst_title "Remove all LXC containers of suite" + rst_para "existing containers matching ${_BGreen}$LXC_HOST_PREFIX-*${_creset}" + echo + lxc list "$LXC_HOST_PREFIX-" + echo -en "\\n${_BRed}LXC containers to delete::${_creset}\\n\\n ${CONTAINERS[*]}\\n" | $FMT + local default=Ny + [[ $FORCE_TIMEOUT = 0 ]] && default=Yn + if ask_yn "Do you really want to delete these containers" $default; then + for i in "${CONTAINERS[@]}"; do + lxc_delete_container "$i" + done + fi + echo + lxc list "$LXC_HOST_PREFIX-" +} + +# images +# ------ + +lxc_copy_images_localy() { + rst_title "copy images" section + for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do + lxc_image_copy "${LXC_SUITE[i]}" "${LXC_SUITE[i+1]}" + done + # lxc image list local: && wait_key +} + +lxc_delete_images_localy() { + rst_title "Delete LXC images" + rst_para "local existing images" + echo + lxc image list local: + echo -en "\\n${_BRed}LXC images to delete::${_creset}\\n\\n ${LOCAL_IMAGES[*]}\\n" + if ask_yn "Do you really want to delete these images"; then + for i in "${LOCAL_IMAGES[@]}"; do + lxc_delete_local_image "$i" + done + fi + + for i in $(lxc image list --format csv | grep '^,' | sed 's/,\([^,]*\).*$/\1/'); do + if ask_yn "Image $i has no alias, do you want to delete the image?" Yn; then + lxc_delete_local_image "$i" + fi + done + + echo + lxc image list local: +} + +show_images(){ + rst_title "local images" + echo + lxc image list local: + echo -en "\\n${_Green}LXC suite images::${_creset}\\n\\n ${LOCAL_IMAGES[*]}\\n" + wait_key + for i in "${LOCAL_IMAGES[@]}"; do + if lxc_image_exists "$i"; then + info_msg "lxc image info ${_BBlue}${i}${_creset}" + lxc image info "$i" | prefix_stdout "[${_BBlue}${i}${_creset}] " + else + warn_msg "image ${_BBlue}$i${_creset} does not yet exists" + fi + done + +} + + +# container +# --------- + +show_suite(){ + rst_title "LXC suite ($LXC_HOST_PREFIX-*)" + echo + lxc list "$LXC_HOST_PREFIX-" + echo + for i in "${CONTAINERS[@]}"; do + if ! lxc_exists "$i"; then + warn_msg "container ${_BBlue}$i${_creset} does not yet exists" + else + lxc exec -t "${i}" -- "${LXC_REPO_ROOT}/utils/lxc.sh" __show suite \ + | prefix_stdout "[${_BBlue}${i}${_creset}] " + echo + fi + done +} + +lxc_cmd() { + for i in "${CONTAINERS[@]}"; do + if ! lxc_exists "$i"; then + warn_msg "container ${_BBlue}$i${_creset} does not yet exists" + else + info_msg "lxc $* $i" + lxc "$@" "$i" | prefix_stdout "[${_BBlue}${i}${_creset}] " + fi + done +} + +lxc_exec_cmd() { + local name="$1" + shift + exit_val= + info_msg "[${_BBlue}${name}${_creset}] ${_BGreen}${*}${_creset}" + lxc exec -t --cwd "${LXC_REPO_ROOT}" "${name}" -- bash -c "$*" + exit_val=$? + if [[ $exit_val -ne 0 ]]; then + warn_msg "[${_BBlue}${name}${_creset}] exit code (${_BRed}${exit_val}${_creset}) from ${_BGreen}${*}${_creset}" + else + info_msg "[${_BBlue}${name}${_creset}] exit code (${exit_val}) from ${_BGreen}${*}${_creset}" + fi +} + +lxc_exec() { + for i in "${CONTAINERS[@]}"; do + if ! lxc_exists "$i"; then + warn_msg "container ${_BBlue}$i${_creset} does not yet exists" + else + lxc_exec_cmd "${i}" "$@" | prefix_stdout "[${_BBlue}${i}${_creset}] " + fi + done +} + +lxc_init_all_containers() { + rst_title "init all containers" section + + local image_name + local container_name + + for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do + lxc_init_container "${LXC_SUITE[i+1]}" "${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}" + done +} + +lxc_config_all_containers() { + rst_title "configure all containers" section + + for i in "${CONTAINERS[@]}"; do + lxc_config_container "${i}" + done +} + +lxc_config_container() { + info_msg "[${_BBlue}$1${_creset}] configure container ..." + + info_msg "[${_BBlue}$1${_creset}] map uid/gid from host to container" + # https://lxd.readthedocs.io/en/latest/userns-idmap/#custom-idmaps + echo -e -n "uid $HOST_USER_ID 0\\ngid $HOST_GROUP_ID 0"\ + | lxc config set "$1" raw.idmap - + + info_msg "[${_BBlue}$1${_creset}] share ${REPO_ROOT} (repo_share) from HOST into container" + # https://lxd.readthedocs.io/en/latest/instances/#type-disk + lxc config device add "$1" repo_share disk \ + source="${REPO_ROOT}" \ + path="${LXC_REPO_ROOT}" &>/dev/null + # lxc config show "$1" && wait_key +} + +lxc_boilerplate_all_containers() { + rst_title "run LXC boilerplate scripts" section + + local boilerplate_script + local image_name + + for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do + + image_name="${LXC_SUITE[i+1]}" + boilerplate_script="${image_name}_boilerplate" + boilerplate_script="${!boilerplate_script}" + + lxc_install_boilerplate "${LXC_HOST_PREFIX}-${image_name}" "$boilerplate_script" + + if [[ -z "${boilerplate_script}" ]]; then + err_msg "[${_BBlue}${container_name}${_creset}] no boilerplate for image '${image_name}'" + fi + done +} + +lxc_install_boilerplate() { + + # usage: lxc_install_boilerplate <container-name> <string: shell commands ..> + # + # usage: lxc_install_boilerplate searx-archlinux "${archlinux_boilerplate}" + + local container_name="$1" + local boilerplate_script="$2" + + info_msg "[${_BBlue}${container_name}${_creset}] init .." + if lxc start -q "${container_name}" &>/dev/null; then + sleep 5 # guest needs some time to come up and get an IP + fi + lxc_init_container_env "${container_name}" + info_msg "[${_BBlue}${container_name}${_creset}] install /.lxcenv.mk .." + cat <<EOF | lxc exec "${container_name}" -- bash | prefix_stdout "[${_BBlue}${container_name}${_creset}] " +rm -f "/.lxcenv.mk" +ln -s "${LXC_REPO_ROOT}/utils/makefile.lxc" "/.lxcenv.mk" +ls -l "/.lxcenv.mk" +EOF + + info_msg "[${_BBlue}${container_name}${_creset}] run LXC boilerplate scripts .." + if lxc start -q "${container_name}" &>/dev/null; then + sleep 5 # guest needs some time to come up and get an IP + fi + if [[ -n "${boilerplate_script}" ]]; then + echo "${boilerplate_script}" \ + | lxc exec "${container_name}" -- bash \ + | prefix_stdout "[${_BBlue}${container_name}${_creset}] " + fi +} + + +# ---------------------------------------------------------------------------- +main "$@" +# ---------------------------------------------------------------------------- diff --git a/utils/makefile.include b/utils/makefile.include index 716889c02..65aca70f0 100644 --- a/utils/makefile.include +++ b/utils/makefile.include @@ -1,12 +1,25 @@ # -*- coding: utf-8; mode: makefile-gmake -*- +ifeq (,$(wildcard /.lxcenv.mk)) +PHONY += lxc-activate lxc-purge +lxc-activate: + @$(MAKE) -s -f /share/searx/utils/makefile.lxc lxc-activate +lxc-purge: + $(Q)rm -rf ./lxc +else + include /.lxcenv.mk +endif + +ifeq (,$(wildcard /.lxcenv.mk)) make-help: +else +make-help: lxc-help +endif @echo ' make V=0|1 [targets] 0 => quiet build (default), 1 => verbose build' @echo ' make V=2 [targets] 2 => give reason for rebuild of target' quiet_cmd_common_clean = CLEAN $@ cmd_common_clean = \ - rm -rf tests/build ;\ find . -name '*.orig' -exec rm -f {} + ;\ find . -name '*.rej' -exec rm -f {} + ;\ find . -name '*~' -exec rm -f {} + ;\ @@ -126,3 +139,4 @@ echo-cmd = $(if $($(quiet)cmd_$(1)),echo '$(call escsq,$($(quiet)cmd_$(1)))$(ech # printing commands cmd = @$(echo-cmd) $(cmd_$(1)) +.PHONY: $(PHONY) diff --git a/utils/makefile.lxc b/utils/makefile.lxc new file mode 100644 index 000000000..fd999e40d --- /dev/null +++ b/utils/makefile.lxc @@ -0,0 +1,29 @@ +# -*- coding: utf-8; mode: makefile-gmake -*- +# +# LXC environment +# =============== +# +# To activate/deactivate LXC makefile environment in a container, set/unset link +# from root '/.lxcenv.mk' to *this* file:: +# +# sudo make ./utils/makefile.lxc lxc-activate +# sudo make ./utils/makefile.lxc lxc-deactivate + +LXC_ENV_FOLDER=lxc/$(shell hostname)/ + +lxc-help:: + @echo 'LXC: running in container LXC_ENV_FOLDER=$(LXC_ENV_FOLDER)' + +# If not activated, serve target 'lxc-activate' .. +ifeq (,$(wildcard /.lxcenv.mk)) +PHONY += lxc-activate +lxc-activate: + ln -s "$(abspath $(lastword $(MAKEFILE_LIST)))" "/.lxcenv.mk" +else +# .. and if activated, serve target 'lxc-deactivate'. +PHONY += lxc-deactivate +lxc-deactivate: + rm /.lxcenv.mk +endif + +.PHONY: $(PHONY) diff --git a/utils/makefile.python b/utils/makefile.python index 590bbdb46..df16acbbf 100644 --- a/utils/makefile.python +++ b/utils/makefile.python @@ -8,9 +8,9 @@ export PYTHONPATH := $(SITE_PYTHON):$$PYTHONPATH export PY_ENV PYDIST PYBUILD # folder where the python distribution takes place -PYDIST ?= ./py_dist +PYDIST = ./$(LXC_ENV_FOLDER)dist # folder where the python intermediate build files take place -PYBUILD ?= ./py_build +PYBUILD = ./$(LXC_ENV_FOLDER)build # python version to use PY ?=3 # $(PYTHON) points to the python interpreter from the OS! The python from the @@ -30,8 +30,7 @@ PYLINT_RC ?= .pylintrc TEST_FOLDER ?= ./tests TEST ?= . -VTENV_OPTS = "--no-site-packages" -PY_ENV = ./local/py$(PY) +PY_ENV = ./$(LXC_ENV_FOLDER)local/py$(PY) PY_ENV_BIN = $(PY_ENV)/bin PY_ENV_ACT = . $(PY_ENV_BIN)/activate @@ -41,6 +40,7 @@ ifeq ($(OS),Windows_NT) PY_ENV_ACT = $(PY_ENV_BIN)/activate endif +VTENV_OPTS ?= ifeq ($(PYTHON),python) VIRTUALENV = virtualenv else diff --git a/utils/makefile.sphinx b/utils/makefile.sphinx index 2c1922fc9..1b0f42ccb 100644 --- a/utils/makefile.sphinx +++ b/utils/makefile.sphinx @@ -1,17 +1,19 @@ # -*- coding: utf-8; mode: makefile-gmake -*- +export DOCS_FOLDER DOCS_BUILD DOCS_DIST BOOKS_FOLDER BOOKS_DIST + # You can set these variables from the command line. SPHINXOPTS ?= SPHINXBUILD ?= $(PY_ENV_BIN)/sphinx-build SPHINX_CONF ?= conf.py -DOCS_FOLDER ?= docs -DOCS_BUILD ?= build/docs -DOCS_DIST ?= dist/docs +DOCS_FOLDER = ./docs +DOCS_BUILD = ./$(LXC_ENV_FOLDER)build/docs +DOCS_DIST = ./$(LXC_ENV_FOLDER)dist/docs GH_PAGES ?= gh-pages -BOOKS_FOLDER ?= docs -BOOKS_DIST ?= dist/books +BOOKS_FOLDER = ./docs +BOOKS_DIST = ./$(LXC_ENV_FOLDER)dist/books ifeq ($(KBUILD_VERBOSE),1) SPHINX_VERBOSE = "-v" @@ -54,11 +56,13 @@ docs-help: # requirements # ------------------------------------------------------------------------------ -sphinx-doc: $(PY_ENV) +sphinx-doc-prebuilds:: $(PY_ENV) + +sphinx-doc: sphinx-doc-prebuilds @echo "PYENV installing Sphinx$(SPHINXVERS)" $(Q)$(PY_ENV_BIN)/pip install $(PIP_VERBOSE) 'Sphinx$(SPHINXVERS)' -sphinx-live: $(PY_ENV) +sphinx-live: sphinx-doc-prebuilds @echo "PYENV installing Sphinx$(SPHINXVERS)" $(Q)$(PY_ENV_BIN)/pip install $(PIP_VERBOSE) 'Sphinx$(SPHINXVERS)' sphinx-autobuild @@ -113,7 +117,7 @@ quiet_cmd_sphinx_clean = CLEAN $@ # targets # ------------------------------------------------------------------------------ -# build PDF of whole documentation in: $(DOCS_DIST)/pdf +# build PDF of whole documentation in: $(DOCS_DIST)/pdf PHONY += sphinx-pdf sphinx-pdf: sphinx-latex @@ -154,7 +158,7 @@ $(BOOKS_HTML): sphinx-doc | $(BOOKS_DIST) -b html \ -c $(DOCS_FOLDER) \ -d $(DOCS_BUILD)/books/$(patsubst books/%.html,%,$@)/.doctrees \ - $(patsubst books/%.html,%,$@) \ + $(BOOKS_FOLDER)/$(patsubst books/%.html,%,$@) \ $(BOOKS_DIST)/$(patsubst books/%.html,%,$@) @echo "SPHINX $@ --> file://$(abspath $(BOOKS_DIST)/$(patsubst books/%.html,%,$@))" @@ -166,7 +170,7 @@ $(BOOKS_LIVE): sphinx-live | $(BOOKS_DIST) -b html \ -c $(DOCS_FOLDER) \ -d $(DOCS_BUILD)/books/$(patsubst books/%.live,%,$@)/.doctrees \ - $(patsubst books/%.live,%,$@) \ + $(BOOKS_FOLDER)/$(patsubst books/%.live,%,$@) \ $(BOOKS_DIST)/$(patsubst books/%.live,%,$@) $(BOOKS_PDF): %.pdf : %.latex @@ -182,7 +186,7 @@ $(BOOKS_LATEX): sphinx-doc | $(BOOKS_DIST) -b latex \ -c $(DOCS_FOLDER) \ -d $(DOCS_BUILD)/books/$(patsubst books/%.latex,%,$@)/.doctrees \ - $(patsubst books/%.latex,%,$@) \ + $(BOOKS_FOLDER)/$(patsubst books/%.latex,%,$@) \ $(DOCS_BUILD)/latex/$(patsubst books/%.latex,%,$@) @echo "SPHINX $@ --> file://$(abspath $(DOCS_BUILD)/latex/$(patsubst books/%.latex,%,$@))" diff --git a/utils/morty.sh b/utils/morty.sh new file mode 100755 index 000000000..75bfeeedf --- /dev/null +++ b/utils/morty.sh @@ -0,0 +1,546 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- +# SPDX-License-Identifier: AGPL-3.0-or-later + +# shellcheck source=utils/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +# shellcheck source=utils/brand.env +source "${REPO_ROOT}/utils/brand.env" +source_dot_config +SEARX_URL="${PUBLIC_URL:-http://$(uname -n)/searx}" +source "${REPO_ROOT}/utils/lxc-searx.env" +in_container && lxc_set_suite_env + +# ---------------------------------------------------------------------------- +# config +# ---------------------------------------------------------------------------- + +MORTY_LISTEN="${MORTY_LISTEN:-127.0.0.1:3000}" +PUBLIC_URL_PATH_MORTY="${PUBLIC_URL_PATH_MORTY:-/morty/}" + +PUBLIC_URL_MORTY="${PUBLIC_URL_MORTY:-$(echo "$SEARX_URL" | sed -e's,^\(.*://[^/]*\).*,\1,g')${PUBLIC_URL_PATH_MORTY}}" + +# shellcheck disable=SC2034 +MORTY_TIMEOUT=5 + +SERVICE_NAME="morty" +SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}" +SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}" +SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}" +SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service" +# shellcheck disable=SC2034 +SERVICE_GROUP="${SERVICE_USER}" +# shellcheck disable=SC2034 +SERVICE_ENV_DEBUG=false + +GO_ENV="${SERVICE_HOME}/.go_env" +GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz" +GO_TAR=$(basename "$GO_PKG_URL") + +# shellcheck disable=SC2034 +CONFIG_FILES=() + +# Apache Settings + +APACHE_MORTY_SITE="morty.conf" +NGINX_MORTY_SITE="morty.conf" + +# ---------------------------------------------------------------------------- +usage() { +# ---------------------------------------------------------------------------- + + # shellcheck disable=SC1117 + cat <<EOF +usage:: + $(basename "$0") shell + $(basename "$0") install [all|user] + $(basename "$0") update [morty] + $(basename "$0") remove [all] + $(basename "$0") activate [service] + $(basename "$0") deactivate [service] + $(basename "$0") inspect [service] + $(basename "$0") option [debug-on|debug-off|new-key] + $(basename "$0") apache [install|remove] + $(basename "$0") nginx [install|remove] + $(basename "$0") info [searx] + +shell + start interactive shell from user ${SERVICE_USER} +install / remove + all: complete setup of morty service + user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME) +update morty + Update morty installation ($SERVICE_HOME) +activate service + activate and start service daemon (systemd unit) +deactivate service + stop and deactivate service daemon (systemd unit) +inspect service + show service status and log +option + set one of the available options + :new-key: set new morty key +apache : ${PUBLIC_URL_MORTY} + :install: apache site with a reverse proxy (ProxyPass) + :remove: apache site ${APACHE_MORTY_SITE} +nginx (${PUBLIC_URL_MORTY}) + :install: nginx site with a reverse proxy (ProxyPass) + :remove: nginx site ${NGINX_MORTY_SITE} + +If needed, set the environment variables in the '${DOT_CONFIG#"$REPO_ROOT/"}' file:: + PUBLIC_URL_MORTY: ${PUBLIC_URL_MORTY} + MORTY_LISTEN: ${MORTY_LISTEN} + SERVICE_USER: ${SERVICE_USER} +EOF + if in_container; then + # in containers the service is listening on 0.0.0.0 (see lxc-searx.env) + for ip in $(global_IPs) ; do + if [[ $ip =~ .*:.* ]]; then + echo " container URL (IPv6): http://[${ip#*|}]:3000/" + else + # IPv4: + echo " container URL (IPv4): http://${ip#*|}:3000/" + fi + done + fi + echo + info_searx + + [[ -n ${1} ]] && err_msg "$1" +} + +info_searx() { + # shellcheck disable=SC1117 + cat <<EOF +To activate result and image proxy in searx, edit settings.yml (read: +${DOCS_URL}/admin/morty.html):: + result_proxy: + url : ${PUBLIC_URL_MORTY} + server: + image_proxy : True +EOF +} + +main() { + required_commands \ + sudo install git wget curl \ + || exit + + local _usage="ERROR: unknown or missing $1 command $2" + + case $1 in + --getenv) var="$2"; echo "${!var}"; exit 0;; + -h|--help) usage; exit 0;; + + shell) + sudo_or_exit + interactive_shell "${SERVICE_USER}" + ;; + inspect) + case $2 in + service) + sudo_or_exit + inspect_service + ;; + *) usage "$_usage"; exit 42;; + esac ;; + install) + rst_title "$SERVICE_NAME" part + sudo_or_exit + case $2 in + all) install_all ;; + user) assert_user ;; + *) usage "$_usage"; exit 42;; + esac ;; + update) + sudo_or_exit + case $2 in + morty) update_morty ;; + *) usage "$_usage"; exit 42;; + esac ;; + remove) + sudo_or_exit + case $2 in + all) remove_all;; + user) drop_service_account "${SERVICE_USER}" ;; + *) usage "$_usage"; exit 42;; + esac ;; + activate) + sudo_or_exit + case $2 in + service) systemd_activate_service "${SERVICE_NAME}" ;; + *) usage "$_usage"; exit 42;; + esac ;; + deactivate) + sudo_or_exit + case $2 in + service) systemd_deactivate_service "${SERVICE_NAME}" ;; + *) usage "$_usage"; exit 42;; + esac ;; + apache) + sudo_or_exit + case $2 in + install) install_apache_site ;; + remove) remove_apache_site ;; + *) usage "$_usage"; exit 42;; + esac ;; + nginx) + sudo_or_exit + case $2 in + install) install_nginx_site ;; + remove) remove_nginx_site ;; + *) usage "$_usage"; exit 42;; + esac ;; + info) + case $2 in + searx) info_searx ;; + *) usage "$_usage"; exit 42;; + esac ;; + option) + sudo_or_exit + case $2 in + new-key) set_new_key ;; + debug-on) enable_debug ;; + debug-off) disable_debug ;; + *) usage "$_usage"; exit 42;; + esac ;; + doc) rst-doc ;; + *) usage "ERROR: unknown or missing command $1"; exit 42;; + esac +} + +install_all() { + + MORTY_KEY="$(head -c 32 /dev/urandom | base64)" + + rst_title "Install $SERVICE_NAME (service)" + assert_user + wait_key + install_go "${GO_PKG_URL}" "${GO_TAR}" "${SERVICE_USER}" + wait_key + install_morty + wait_key + systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" + wait_key + if ! service_is_available "http://${MORTY_LISTEN}" ; then + err_msg "Morty does not listening on: http://${MORTY_LISTEN}" + fi + if apache_is_installed; then + info_msg "Apache is installed on this host." + if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then + install_apache_site + fi + elif nginx_is_installed; then + info_msg "nginx is installed on this host." + if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then + install_nginx_site + fi + fi + info_searx + if ask_yn "Add image and result proxy to searx settings.yml?" Yn; then + "${REPO_ROOT}/utils/searx.sh" option result-proxy "${PUBLIC_URL_MORTY}" "${MORTY_KEY}" + "${REPO_ROOT}/utils/searx.sh" option image-proxy-on + fi + + if ask_yn "Do you want to inspect the installation?" Ny; then + inspect_service + fi + +} + +remove_all() { + rst_title "De-Install $SERVICE_NAME (service)" + + rst_para "\ +It goes without saying that this script can only be used to remove +installations that were installed with this script." + + if systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then + drop_service_account "${SERVICE_USER}" + fi +} + +assert_user() { + rst_title "user $SERVICE_USER" section + echo + tee_stderr 1 <<EOF | bash | prefix_stdout +useradd --shell /bin/bash --system \ + --home-dir "$SERVICE_HOME" \ + --comment 'Web content sanitizer proxy' $SERVICE_USER +mkdir "$SERVICE_HOME" +chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME" +groups $SERVICE_USER +EOF + SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)" + export SERVICE_HOME + echo "export SERVICE_HOME=$SERVICE_HOME" + + cat > "$GO_ENV" <<EOF +export GOPATH=\$HOME/go-apps +export PATH=\$PATH:\$HOME/local/go/bin:\$GOPATH/bin +EOF + echo "Environment $GO_ENV has been setup." + + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" +grep -qFs -- 'source $GO_ENV' ~/.profile || echo 'source $GO_ENV' >> ~/.profile +EOF +} + +morty_is_installed() { + [[ -f $SERVICE_HOME/go-apps/bin/morty ]] +} + +_svcpr=" ${_Yellow}|${SERVICE_USER}|${_creset} " + +install_morty() { + rst_title "Install morty in user's ~/go-apps" section + echo + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr" +go get -v -u github.com/asciimoo/morty +EOF + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr" +cd \$GOPATH/src/github.com/asciimoo/morty +go test +go test -benchmem -bench . +EOF +} + +update_morty() { + rst_title "Update morty" section + echo + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr" +go get -v -u github.com/asciimoo/morty +EOF + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr" +cd \$GOPATH/src/github.com/asciimoo/morty +go test +go test -benchmem -bench . +EOF +} + +set_service_env_debug() { + + # usage: set_service_env_debug [false|true] + + # shellcheck disable=SC2034 + local SERVICE_ENV_DEBUG="${1:-false}" + if systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then + systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" + fi +} + +inspect_service() { + + rst_title "service status & log" + + cat <<EOF + +sourced ${DOT_CONFIG#"$REPO_ROOT/"} : + + MORTY_LISTEN : ${MORTY_LISTEN} + +EOF + + if service_account_is_available "$SERVICE_USER"; then + info_msg "service account $SERVICE_USER available." + else + err_msg "service account $SERVICE_USER not available!" + fi + if go_is_available "$SERVICE_USER"; then + info_msg "~$SERVICE_USER: go is installed" + else + err_msg "~$SERVICE_USER: go is not installed" + fi + if morty_is_installed; then + info_msg "~$SERVICE_USER: morty app is installed" + else + err_msg "~$SERVICE_USER: morty app is not installed!" + fi + + if ! service_is_available "http://${MORTY_LISTEN}" ; then + err_msg "Morty does not listening on: http://${MORTY_LISTEN}" + echo -e "${_Green}stop with [${_BCyan}CTRL-C${_Green}] or .." + wait_key + fi + + if ! service_is_available "${PUBLIC_URL_MORTY}"; then + warn_msg "Public service at ${PUBLIC_URL_MORTY} is not available!" + if ! in_container; then + warn_msg "Check if public name is correct and routed or use the public IP from above." + fi + fi + + if in_container; then + lxc_suite_info + else + info_msg "public URL --> ${PUBLIC_URL_MORTY}" + info_msg "morty URL --> http://${MORTY_LISTEN}" + fi + + local _debug_on + if ask_yn "Enable morty debug mode (needs reinstall of systemd service)?"; then + enable_debug + _debug_on=1 + else + systemctl --no-pager -l status "${SERVICE_NAME}" + fi + echo + + # shellcheck disable=SC2059 + printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log" + read -r -s -n1 -t 5 + echo + while true; do + trap break 2 + journalctl -f -u "${SERVICE_NAME}" + done + + if [[ $_debug_on == 1 ]]; then + FORCE_SELECTION=Y disable_debug + fi + return 0 +} + +enable_debug() { + warn_msg "Do not enable debug in production enviroments!!" + info_msg "Enabling debug option needs to reinstall systemd service!" + set_service_env_debug true +} + +disable_debug() { + info_msg "Disabling debug option needs to reinstall systemd service!" + set_service_env_debug false +} + + +set_new_key() { + rst_title "Set morty key" + echo + + MORTY_KEY="$(head -c 32 /dev/urandom | base64)" + info_msg "morty key: '${MORTY_KEY}'" + + warn_msg "this will need to reinstall services .." + MSG="${_Green}press any [${_BCyan}KEY${_Green}] to continue // stop with [${_BCyan}CTRL-C${_creset}]" wait_key + + systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" + "${REPO_ROOT}/utils/searx.sh" option result-proxy "${PUBLIC_URL_MORTY}" "${MORTY_KEY}" + "${REPO_ROOT}/utils/searx.sh" option image-proxy-on +} + + +install_apache_site() { + + rst_title "Install Apache site $APACHE_MORTY_SITE" + + rst_para "\ +This installs a reverse proxy (ProxyPass) into apache site (${APACHE_MORTY_SITE})" + + ! apache_is_installed && err_msg "Apache is not installed." + + if ! ask_yn "Do you really want to continue?" Yn; then + return + else + install_apache + fi + + apache_install_site "${APACHE_MORTY_SITE}" + + info_msg "testing public url .." + if ! service_is_available "${PUBLIC_URL_MORTY}"; then + err_msg "Public service at ${PUBLIC_URL_MORTY} is not available!" + fi +} + +remove_apache_site() { + + rst_title "Remove Apache site $APACHE_MORTY_SITE" + + rst_para "\ +This removes apache site ${APACHE_MORTY_SITE}." + + ! apache_is_installed && err_msg "Apache is not installed." + + if ! ask_yn "Do you really want to continue?" Yn; then + return + fi + + apache_remove_site "$APACHE_MORTY_SITE" +} + +install_nginx_site() { + + rst_title "Install nginx site $NGINX_MORTY_SITE" + + rst_para "\ +This installs a reverse proxy (ProxyPass) into nginx site (${NGINX_MORTY_SITE})" + + ! nginx_is_installed && err_msg "nginx is not installed." + + if ! ask_yn "Do you really want to continue?" Yn; then + return + else + install_nginx + fi + + "${REPO_ROOT}/utils/searx.sh" install uwsgi + + # shellcheck disable=SC2034 + SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC) + # shellcheck disable=SC2034 + SEARX_URL_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_URL_PATH) + nginx_install_app "${NGINX_MORTY_SITE}" + + info_msg "testing public url .." + if ! service_is_available "${PUBLIC_URL_MORTY}"; then + err_msg "Public service at ${PUBLIC_URL_MORTY} is not available!" + fi +} + +remove_nginx_site() { + + rst_title "Remove nginx site $NGINX_MORTY_SITE" + + rst_para "\ +This removes nginx site ${NGINX_MORTY_SITE}." + + ! nginx_is_installed && err_msg "nginx is not installed." + + if ! ask_yn "Do you really want to continue?" Yn; then + return + fi + + nginx_remove_site "$NGINX_MORTY_SITE" + +} + +rst-doc() { + + eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/morty.rst")\"" + + echo -e "\n.. START install systemd unit" + cat <<EOF +.. tabs:: + + .. group-tab:: systemd + + .. code:: bash + +EOF + eval "echo \"$(< "${TEMPLATES}/${SERVICE_SYSTEMD_UNIT}")\"" | prefix_stdout " " + echo -e "\n.. END install systemd unit" + + # for DIST_NAME in ubuntu-20.04 arch fedora; do + # ( + # DIST_ID=${DIST_NAME%-*} + # DIST_VERS=${DIST_NAME#*-} + # [[ $DIST_VERS =~ $DIST_ID ]] && DIST_VERS= + # # ... + # ) + # done +} + + +# ---------------------------------------------------------------------------- +main "$@" +# ---------------------------------------------------------------------------- diff --git a/utils/searx.sh b/utils/searx.sh new file mode 100755 index 000000000..4009fddb0 --- /dev/null +++ b/utils/searx.sh @@ -0,0 +1,869 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- +# SPDX-License-Identifier: AGPL-3.0-or-later +# shellcheck disable=SC2001 + +# shellcheck source=utils/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +# shellcheck source=utils/brand.env +source "${REPO_ROOT}/utils/brand.env" +source_dot_config +source "${REPO_ROOT}/utils/lxc-searx.env" +in_container && lxc_set_suite_env + +# ---------------------------------------------------------------------------- +# config +# ---------------------------------------------------------------------------- + +PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}" + +SEARX_INTERNAL_HTTP="${SEARX_INTERNAL_HTTP:-127.0.0.1:8888}" + +SEARX_URL_PATH="${SEARX_URL_PATH:-$(echo "${PUBLIC_URL}" \ +| sed -e 's,^.*://[^/]*\(/.*\),\1,g')}" +[[ "${SEARX_URL_PATH}" == "${PUBLIC_URL}" ]] && SEARX_URL_PATH=/ +SEARX_INSTANCE_NAME="${SEARX_INSTANCE_NAME:-searx@$(echo "$PUBLIC_URL" \ +| sed -e 's,^.*://\([^\:/]*\).*,\1,g') }" + +SERVICE_NAME="searx" +SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}" +SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}" +SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}" +# shellcheck disable=SC2034 +SERVICE_GROUP="${SERVICE_USER}" + +GIT_BRANCH="${GIT_BRANCH:-master}" +SEARX_PYENV="${SERVICE_HOME}/searx-pyenv" +SEARX_SRC="${SERVICE_HOME}/searx-src" +SEARX_SETTINGS_PATH="/etc/searx/settings.yml" +SEARX_UWSGI_APP="searx.ini" +# shellcheck disable=SC2034 +SEARX_UWSGI_SOCKET="/run/uwsgi/app/searx/socket" + +# apt packages +SEARX_PACKAGES_debian="\ +virtualenv python3-dev python3-babel python3-venv +uwsgi uwsgi-plugin-python3 +git build-essential libxslt-dev zlib1g-dev libffi-dev libssl-dev +shellcheck" + +BUILD_PACKAGES_debian="\ +firefox graphviz imagemagick texlive-xetex librsvg2-bin +texlive-latex-recommended texlive-extra-utils ttf-dejavu +latexmk" + +# pacman packages +SEARX_PACKAGES_arch="\ +python-virtualenv python python-pip python-lxml python-babel +uwsgi uwsgi-plugin-python +git base-devel libxml2 +shellcheck" + +BUILD_PACKAGES_arch="\ +firefox graphviz imagemagick texlive-bin extra/librsvg +texlive-core texlive-latexextra ttf-dejavu" + +# dnf packages +SEARX_PACKAGES_fedora="\ +virtualenv python python-pip python-lxml python-babel +uwsgi uwsgi-plugin-python3 +git @development-tools libxml2 +ShellCheck" + +BUILD_PACKAGES_fedora="\ +firefox graphviz graphviz-gd ImageMagick librsvg2-tools +texlive-xetex-bin texlive-collection-fontsrecommended +texlive-collection-latex dejavu-sans-fonts dejavu-serif-fonts +dejavu-sans-mono-fonts" + +case $DIST_ID-$DIST_VERS in + ubuntu-16.04|ubuntu-18.04) + SEARX_PACKAGES="${SEARX_PACKAGES_debian}" + BUILD_PACKAGES="${BUILD_PACKAGES_debian}" + APACHE_PACKAGES="$APACHE_PACKAGES libapache2-mod-proxy-uwsgi" + ;; + ubuntu-20.04) + # https://askubuntu.com/a/1224710 + SEARX_PACKAGES="${SEARX_PACKAGES_debian} python-is-python3" + BUILD_PACKAGES="${BUILD_PACKAGES_debian}" + ;; + ubuntu-*|debian-*) + SEARX_PACKAGES="${SEARX_PACKAGES_debian}" + BUILD_PACKAGES="${BUILD_PACKAGES_debian}" + ;; + arch-*) + SEARX_PACKAGES="${SEARX_PACKAGES_arch}" + BUILD_PACKAGES="${BUILD_PACKAGES_arch}" + ;; + fedora-*) + SEARX_PACKAGES="${SEARX_PACKAGES_fedora}" + BUILD_PACKAGES="${BUILD_PACKAGES_fedora}" + ;; +esac + +# Apache Settings +APACHE_SEARX_SITE="searx.conf" + +# shellcheck disable=SC2034 +CONFIG_FILES=( + "${uWSGI_APPS_AVAILABLE}/${SEARX_UWSGI_APP}" +) + +# shellcheck disable=SC2034 +CONFIG_BACKUP_ENCRYPTED=( + "${SEARX_SETTINGS_PATH}" +) + +# ---------------------------------------------------------------------------- +usage() { +# ---------------------------------------------------------------------------- + + # shellcheck disable=SC1117 + cat <<EOF +usage:: + $(basename "$0") shell + $(basename "$0") install [all|user|searx-src|pyenv|uwsgi|packages|buildhost] + $(basename "$0") update [searx] + $(basename "$0") remove [all|user|pyenv|searx-src] + $(basename "$0") activate [service] + $(basename "$0") deactivate [service] + $(basename "$0") inspect [service] + $(basename "$0") option [debug-[on|off]|image-proxy-[on|off]|result-proxy <url> <key>] + $(basename "$0") apache [install|remove] + +shell + start interactive shell from user ${SERVICE_USER} +install / remove + :all: complete (de-) installation of searx service + :user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME) + :searx-src: clone $GIT_URL + :pyenv: create/remove virtualenv (python) in $SEARX_PYENV + :uwsgi: install searx uWSGI application + :settings: reinstall settings from ${REPO_ROOT}/searx/settings.yml + :packages: install needed packages from OS package manager + :buildhost: install packages from OS package manager needed by buildhosts +update searx + Update searx installation ($SERVICE_HOME) +activate service + activate and start service daemon (systemd unit) +deactivate service + stop and deactivate service daemon (systemd unit) +inspect service + run some small tests and inspect service's status and log +option + set one of the available options +apache + :install: apache site with the searx uwsgi app + :remove: apache site ${APACHE_FILTRON_SITE} + +searx settings: ${SEARX_SETTINGS_PATH} + +If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/"}' file:: + PUBLIC_URL : ${PUBLIC_URL} + SEARX_INSTANCE_NAME : ${SEARX_INSTANCE_NAME} + SERVICE_USER : ${SERVICE_USER} + SEARX_INTERNAL_HTTP : http://${SEARX_INTERNAL_HTTP} +EOF + if in_container; then + # searx is listening on 127.0.0.1 and not available from outside container + # in containers the service is listening on 0.0.0.0 (see lxc-searx.env) + echo -e "${_BBlack}HINT:${_creset} searx only listen on loopback device" \ + "${_BBlack}inside${_creset} the container." + for ip in $(global_IPs) ; do + if [[ $ip =~ .*:.* ]]; then + echo " container (IPv6): [${ip#*|}]" + else + # IPv4: + echo " container (IPv4): ${ip#*|}" + fi + done + fi + [[ -n ${1} ]] && err_msg "$1" +} + +main() { + required_commands \ + sudo systemctl install git wget curl \ + || exit + + local _usage="unknown or missing $1 command $2" + + case $1 in + --getenv) var="$2"; echo "${!var}"; exit 0;; + -h|--help) usage; exit 0;; + shell) + sudo_or_exit + interactive_shell "${SERVICE_USER}" + ;; + inspect) + case $2 in + service) + sudo_or_exit + inspect_service + ;; + *) usage "$_usage"; exit 42;; + esac ;; + install) + rst_title "$SEARX_INSTANCE_NAME" part + sudo_or_exit + case $2 in + all) install_all ;; + user) assert_user ;; + pyenv) create_pyenv ;; + searx-src) clone_searx ;; + settings) install_settings ;; + uwsgi) + install_searx_uwsgi + if ! service_is_available "http://${SEARX_INTERNAL_HTTP}"; then + err_msg "URL http://${SEARX_INTERNAL_HTTP} not available, check searx & uwsgi setup!" + fi + ;; + packages) + pkg_install "$SEARX_PACKAGES" + ;; + buildhost) + pkg_install "$SEARX_PACKAGES" + pkg_install "$BUILD_PACKAGES" + ;; + *) usage "$_usage"; exit 42;; + esac ;; + update) + sudo_or_exit + case $2 in + searx) update_searx;; + *) usage "$_usage"; exit 42;; + esac ;; + remove) + sudo_or_exit + case $2 in + all) remove_all;; + user) drop_service_account "${SERVICE_USER}";; + pyenv) remove_pyenv ;; + searx-src) remove_searx ;; + *) usage "$_usage"; exit 42;; + esac ;; + activate) + sudo_or_exit + case $2 in + service) + activate_service ;; + *) usage "$_usage"; exit 42;; + esac ;; + deactivate) + sudo_or_exit + case $2 in + service) deactivate_service ;; + *) usage "$_usage"; exit 42;; + esac ;; + option) + sudo_or_exit + case $2 in + debug-on) echo; enable_debug ;; + debug-off) echo; disable_debug ;; + result-proxy) set_result_proxy "$3" "$4" ;; + image-proxy-on) enable_image_proxy ;; + image-proxy-off) disable_image_proxy ;; + *) usage "$_usage"; exit 42;; + esac ;; + apache) + sudo_or_exit + case $2 in + install) install_apache_site ;; + remove) remove_apache_site ;; + *) usage "$_usage"; exit 42;; + esac ;; + doc) rst-doc;; + *) usage "unknown or missing command $1"; exit 42;; + esac +} + +_service_prefix=" ${_Yellow}|$SERVICE_USER|${_creset} " + +install_all() { + rst_title "Install $SEARX_INSTANCE_NAME (service)" + pkg_install "$SEARX_PACKAGES" + wait_key + assert_user + wait_key + clone_searx + wait_key + create_pyenv + wait_key + install_settings + wait_key + test_local_searx + wait_key + install_searx_uwsgi + if ! service_is_available "http://${SEARX_INTERNAL_HTTP}"; then + err_msg "URL http://${SEARX_INTERNAL_HTTP} not available, check searx & uwsgi setup!" + fi + if ask_yn "Do you want to inspect the installation?" Ny; then + inspect_service + fi +} + +update_searx() { + rst_title "Update searx instance" + + echo + tee_stderr 0.3 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +git checkout -B "$GIT_BRANCH" +git pull +pip install -U pip +pip install -U setuptools +pip install -U wheel +pip install -U -e . +EOF + install_settings + uWSGI_restart "$SEARX_UWSGI_APP" +} + +remove_all() { + rst_title "De-Install $SEARX_INSTANCE_NAME (service)" + + rst_para "\ +It goes without saying that this script can only be used to remove +installations that were installed with this script." + + if ! ask_yn "Do you really want to deinstall $SEARX_INSTANCE_NAME?"; then + return + fi + remove_searx_uwsgi + drop_service_account "${SERVICE_USER}" + remove_settings + wait_key + if service_is_available "${PUBLIC_URL}"; then + MSG="** Don't forgett to remove your public site! (${PUBLIC_URL}) **" wait_key 10 + fi +} + +assert_user() { + rst_title "user $SERVICE_USER" section + echo + tee_stderr 1 <<EOF | bash | prefix_stdout +useradd --shell /bin/bash --system \ + --home-dir "$SERVICE_HOME" \ + --comment 'Privacy-respecting metasearch engine' $SERVICE_USER +mkdir "$SERVICE_HOME" +chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME" +groups $SERVICE_USER +EOF + #SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)" + #export SERVICE_HOME + #echo "export SERVICE_HOME=$SERVICE_HOME" +} + +clone_is_available() { + [[ -f "$SEARX_SRC/.git/config" ]] +} + +# shellcheck disable=SC2164 +clone_searx() { + rst_title "Clone searx sources" section + echo + if ! sudo -i -u "$SERVICE_USER" ls -d "$REPO_ROOT" > /dev/null; then + die 42 "user '$SERVICE_USER' missed read permission: $REPO_ROOT" + fi + SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME 2>/dev/null)" + if [[ ! "${SERVICE_HOME}" ]]; then + err_msg "to clone searx sources, user $SERVICE_USER hast to be created first" + return 42 + fi + export SERVICE_HOME + git_clone "$REPO_ROOT" "$SEARX_SRC" \ + "$GIT_BRANCH" "$SERVICE_USER" + + pushd "${SEARX_SRC}" > /dev/null + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +cd "${SEARX_SRC}" +git remote set-url origin ${GIT_URL} +git config user.email "$ADMIN_EMAIL" +git config user.name "$ADMIN_NAME" +git config --list +EOF + popd > /dev/null +} + +install_settings() { + rst_title "${SEARX_SETTINGS_PATH}" section + if ! clone_is_available; then + err_msg "you have to install searx first" + exit 42 + fi + mkdir -p "$(dirname ${SEARX_SETTINGS_PATH})" + + if [[ ! -f ${SEARX_SETTINGS_PATH} ]]; then + info_msg "install settings ${REPO_ROOT}/searx/settings.yml" + info_msg " --> ${SEARX_SETTINGS_PATH}" + cp "${REPO_ROOT}/searx/settings.yml" "${SEARX_SETTINGS_PATH}" + configure_searx + return + fi + + rst_para "Diff between origin's setting file (+) and current (-):" + echo + $DIFF_CMD "${SEARX_SETTINGS_PATH}" "${SEARX_SRC}/searx/settings.yml" + + local action + choose_one action "What should happen to the settings file? " \ + "keep configuration unchanged" \ + "use origin settings" \ + "start interactiv shell" + case $action in + "keep configuration unchanged") + info_msg "leave settings file unchanged" + ;; + "use origin settings") + backup_file "${SEARX_SETTINGS_PATH}" + info_msg "install origin settings" + cp "${SEARX_SRC}/searx/settings.yml" "${SEARX_SETTINGS_PATH}" + ;; + "start interactiv shell") + backup_file "${SEARX_SETTINGS_PATH}" + echo -e "// exit with [${_BCyan}CTRL-D${_creset}]" + sudo -H -i + rst_para 'Diff between new setting file (-) and current (+):' + echo + $DIFF_CMD "${SEARX_SRC}/searx/settings.yml" "${SEARX_SETTINGS_PATH}" + wait_key + ;; + esac +} + +remove_settings() { + rst_title "remove searx settings" section + echo + info_msg "delete ${SEARX_SETTINGS_PATH}" + rm -f "${SEARX_SETTINGS_PATH}" +} + +remove_searx() { + rst_title "Drop searx sources" section + if ask_yn "Do you really want to drop searx sources ($SEARX_SRC)?"; then + rm -rf "$SEARX_SRC" + else + rst_para "Leave searx sources unchanged." + fi +} + +pyenv_is_available() { + [[ -f "${SEARX_PYENV}/bin/activate" ]] +} + +create_pyenv() { + rst_title "Create virtualenv (python)" section + echo + if [[ ! -f "${SEARX_SRC}/manage.sh" ]]; then + err_msg "to create pyenv for searx, searx has to be cloned first" + return 42 + fi + info_msg "create pyenv in ${SEARX_PYENV}" + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +rm -rf "${SEARX_PYENV}" +python3 -m venv "${SEARX_PYENV}" +grep -qFs -- 'source ${SEARX_PYENV}/bin/activate' ~/.profile \ + || echo 'source ${SEARX_PYENV}/bin/activate' >> ~/.profile +EOF + info_msg "inspect python's virtual environment" + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +command -v python && python --version +EOF + wait_key + info_msg "install needed python packages" + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +pip install -U pip +pip install -U setuptools +pip install -U wheel +pip install -U -e . +cd ${SEARX_SRC} +pip install -e . +EOF +} + +remove_pyenv() { + rst_title "Remove virtualenv (python)" section + if ! ask_yn "Do you really want to drop ${SEARX_PYENV} ?"; then + return + fi + info_msg "remove pyenv activation from ~/.profile" + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +grep -v 'source ${SEARX_PYENV}/bin/activate' ~/.profile > ~/.profile.## +mv ~/.profile.## ~/.profile +EOF + rm -rf "${SEARX_PYENV}" +} + +configure_searx() { + rst_title "Configure searx" section + rst_para "Setup searx config located at $SEARX_SETTINGS_PATH" + echo + tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +sed -i -e "s/ultrasecretkey/$(openssl rand -hex 16)/g" "$SEARX_SETTINGS_PATH" +sed -i -e "s/{instance_name}/${SEARX_INSTANCE_NAME}/g" "$SEARX_SETTINGS_PATH" +EOF +} + +test_local_searx() { + rst_title "Testing searx instance localy" section + echo + + if service_is_available "http://${SEARX_INTERNAL_HTTP}" &>/dev/null; then + err_msg "URL/port http://${SEARX_INTERNAL_HTTP} is already in use, you" + err_msg "should stop that service before starting local tests!" + if ! ask_yn "Continue with local tests?"; then + return + fi + fi + sed -i -e "s/debug : False/debug : True/g" "$SEARX_SETTINGS_PATH" + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +export SEARX_SETTINGS_PATH="${SEARX_SETTINGS_PATH}" +cd ${SEARX_SRC} +timeout 10 python searx/webapp.py & +sleep 3 +curl --location --verbose --head --insecure $SEARX_INTERNAL_HTTP +EOF + sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS_PATH" +} + +install_searx_uwsgi() { + rst_title "Install searx's uWSGI app (searx.ini)" section + echo + install_uwsgi + uWSGI_install_app "$SEARX_UWSGI_APP" +} + +remove_searx_uwsgi() { + rst_title "Remove searx's uWSGI app (searx.ini)" section + echo + uWSGI_remove_app "$SEARX_UWSGI_APP" +} + +activate_service() { + rst_title "Activate $SEARX_INSTANCE_NAME (service)" section + echo + uWSGI_enable_app "$SEARX_UWSGI_APP" + uWSGI_restart "$SEARX_UWSGI_APP" +} + +deactivate_service() { + rst_title "De-Activate $SEARX_INSTANCE_NAME (service)" section + echo + uWSGI_disable_app "$SEARX_UWSGI_APP" + uWSGI_restart "$SEARX_UWSGI_APP" +} + +enable_image_proxy() { + info_msg "try to enable image_proxy ..." + tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +sed -i -e "s/image_proxy : False/image_proxy : True/g" "$SEARX_SETTINGS_PATH" +EOF + uWSGI_restart "$SEARX_UWSGI_APP" +} + +disable_image_proxy() { + info_msg "try to enable image_proxy ..." + tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +sed -i -e "s/image_proxy : True/image_proxy : False/g" "$SEARX_SETTINGS_PATH" +EOF + uWSGI_restart "$SEARX_UWSGI_APP" +} + +enable_debug() { + warn_msg "Do not enable debug in production enviroments!!" + info_msg "try to enable debug mode ..." + tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +sed -i -e "s/debug : False/debug : True/g" "$SEARX_SETTINGS_PATH" +EOF + uWSGI_restart "$SEARX_UWSGI_APP" +} + +disable_debug() { + info_msg "try to disable debug mode ..." + tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS_PATH" +EOF + uWSGI_restart "$SEARX_UWSGI_APP" +} + +set_result_proxy() { + + # usage: set_result_proxy <URL> [<key>] + + info_msg "try to set result proxy: '$1' ($2)" + cp "${SEARX_SETTINGS_PATH}" "${SEARX_SETTINGS_PATH}.bak" + _set_result_proxy "$1" "$2" > "${SEARX_SETTINGS_PATH}" +} + +_set_result_proxy() { + local line + local stage=0 + local url=" url: $1" + local key=" key: !!binary \"$2\"" + if [[ -z $2 ]]; then + key= + fi + + while IFS= read -r line + do + if [[ $stage = 0 ]] || [[ $stage = 2 ]] ; then + if [[ $line =~ ^[[:space:]]*#*[[:space:]]*result_proxy[[:space:]]*:[[:space:]]*$ ]]; then + if [[ $stage = 0 ]]; then + stage=1 + echo "result_proxy:" + continue + elif [[ $stage = 2 ]]; then + continue + fi + fi + fi + if [[ $stage = 1 ]] || [[ $stage = 2 ]] ; then + if [[ $line =~ ^[[:space:]]*#*[[:space:]]*url[[:space:]]*:[[:space:]] ]]; then + [[ $stage = 1 ]] && echo "$url" + continue + elif [[ $line =~ ^[[:space:]]*#*[[:space:]]*key[[:space:]]*:[[:space:]] ]]; then + [[ $stage = 1 ]] && [[ -n $key ]] && echo "$key" + continue + elif [[ $line =~ ^[[:space:]]*$ ]]; then + stage=2 + fi + fi + echo "$line" + done < "${SEARX_SETTINGS_PATH}.bak" +} + +function has_substring() { + [[ "$1" != "${2/$1/}" ]] +} +inspect_service() { + rst_title "service status & log" + cat <<EOF + +sourced ${DOT_CONFIG#"$REPO_ROOT/"} : + + PUBLIC_URL : ${PUBLIC_URL} + SEARX_URL_PATH : ${SEARX_URL_PATH} + SEARX_INSTANCE_NAME : ${SEARX_INSTANCE_NAME} + SEARX_INTERNAL_HTTP : ${SEARX_INTERNAL_HTTP} + +EOF + + if service_account_is_available "$SERVICE_USER"; then + info_msg "Service account $SERVICE_USER exists." + else + err_msg "Service account $SERVICE_USER does not exists!" + fi + + if pyenv_is_available; then + info_msg "~$SERVICE_USER: python environment is available." + else + err_msg "~$SERVICE_USER: python environment is not available!" + fi + + if clone_is_available; then + info_msg "~$SERVICE_USER: Searx software is installed." + else + err_msg "~$SERVICE_USER: Missing searx software!" + fi + + if uWSGI_app_enabled "$SEARX_UWSGI_APP"; then + info_msg "uWSGI app $SEARX_UWSGI_APP is enabled." + else + err_msg "uWSGI app $SEARX_UWSGI_APP not enabled!" + fi + + uWSGI_app_available "$SEARX_UWSGI_APP" \ + || err_msg "uWSGI app $SEARX_UWSGI_APP not available!" + + if in_container; then + lxc_suite_info + else + info_msg "public URL --> ${PUBLIC_URL}" + info_msg "internal URL --> http://${SEARX_INTERNAL_HTTP}" + fi + + if ! service_is_available "http://${SEARX_INTERNAL_HTTP}"; then + err_msg "uWSGI app (service) at http://${SEARX_INTERNAL_HTTP} is not available!" + MSG="${_Green}[${_BCyan}CTRL-C${_Green}] to stop or [${_BCyan}KEY${_Green}] to continue"\ + wait_key + fi + + if ! service_is_available "${PUBLIC_URL}"; then + warn_msg "Public service at ${PUBLIC_URL} is not available!" + if ! in_container; then + warn_msg "Check if public name is correct and routed or use the public IP from above." + fi + fi + + local _debug_on + if ask_yn "Enable searx debug mode?"; then + enable_debug + _debug_on=1 + fi + echo + + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) + systemctl --no-pager -l status "${SERVICE_NAME}" + ;; + arch-*) + systemctl --no-pager -l status "uwsgi@${SERVICE_NAME%.*}" + ;; + fedora-*) + systemctl --no-pager -l status uwsgi + ;; + esac + + # shellcheck disable=SC2059 + printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log" + read -r -s -n1 -t 5 + echo + + while true; do + trap break 2 + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) tail -f /var/log/uwsgi/app/searx.log ;; + arch-*) journalctl -f -u "uwsgi@${SERVICE_NAME%.*}" ;; + fedora-*) journalctl -f -u uwsgi ;; + esac + done + + if [[ $_debug_on == 1 ]]; then + disable_debug + fi + return 0 +} + +install_apache_site() { + rst_title "Install Apache site $APACHE_SEARX_SITE" + + rst_para "\ +This installs the searx uwsgi app as apache site. If your server is public to +the internet, you should instead use a reverse proxy (filtron) to block +excessively bot queries." + + ! apache_is_installed && err_msg "Apache is not installed." + + if ! ask_yn "Do you really want to continue?" Yn; then + return + else + install_apache + fi + + apache_install_site --variant=uwsgi "${APACHE_SEARX_SITE}" + + rst_title "Install searx's uWSGI app (searx.ini)" section + echo + uWSGI_install_app --variant=socket "$SEARX_UWSGI_APP" + + if ! service_is_available "${PUBLIC_URL}"; then + err_msg "Public service at ${PUBLIC_URL} is not available!" + fi +} + +remove_apache_site() { + + rst_title "Remove Apache site ${APACHE_SEARX_SITE}" + + rst_para "\ +This removes apache site ${APACHE_SEARX_SITE}." + + ! apache_is_installed && err_msg "Apache is not installed." + + if ! ask_yn "Do you really want to continue?" Yn; then + return + fi + + apache_remove_site "${APACHE_SEARX_SITE}" + + rst_title "Remove searx's uWSGI app (searx.ini)" section + echo + uWSGI_remove_app "$SEARX_UWSGI_APP" +} + +rst-doc() { + local debian="${SEARX_PACKAGES_debian}" + local arch="${SEARX_PACKAGES_arch}" + local fedora="${SEARX_PACKAGES_fedora}" + local debian_build="${BUILD_PACKAGES_debian}" + local arch_build="${BUILD_PACKAGES_arch}" + local fedora_build="${BUILD_PACKAGES_fedora}" + debian="$(echo "${debian}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + arch="$(echo "${arch}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + fedora="$(echo "${fedora}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + debian_build="$(echo "${debian_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + arch_build="$(echo "${arch_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + fedora_build="$(echo "${fedora_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + + eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/searx.rst")\"" + + # I use ubuntu-20.04 here to demonstrate that versions are also suported, + # normaly debian-* and ubuntu-* are most the same. + + for DIST_NAME in ubuntu-20.04 arch fedora; do + ( + DIST_ID=${DIST_NAME%-*} + DIST_VERS=${DIST_NAME#*-} + [[ $DIST_VERS =~ $DIST_ID ]] && DIST_VERS= + uWSGI_distro_setup + + echo -e "\n.. START searx uwsgi-description $DIST_NAME" + + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) cat <<EOF +# init.d --> /usr/share/doc/uwsgi/README.Debian.gz +# For uWSGI debian uses the LSB init process, this might be changed +# one day, see https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=833067 + +create ${uWSGI_APPS_AVAILABLE}/${SEARX_UWSGI_APP} +enable: sudo -H ln -s ${uWSGI_APPS_AVAILABLE}/${SEARX_UWSGI_APP} ${uWSGI_APPS_ENABLED}/ +start: sudo -H service uwsgi start ${SEARX_UWSGI_APP%.*} +restart: sudo -H service uwsgi restart ${SEARX_UWSGI_APP%.*} +stop: sudo -H service uwsgi stop ${SEARX_UWSGI_APP%.*} +disable: sudo -H rm ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP} +EOF + ;; + arch-*) cat <<EOF +# systemd --> /usr/lib/systemd/system/uwsgi@.service +# For uWSGI archlinux uses systemd template units, see +# - http://0pointer.de/blog/projects/instances.html +# - https://uwsgi-docs.readthedocs.io/en/latest/Systemd.html#one-service-per-app-in-systemd + +create: ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP} +enable: sudo -H systemctl enable uwsgi@${SEARX_UWSGI_APP%.*} +start: sudo -H systemctl start uwsgi@${SEARX_UWSGI_APP%.*} +restart: sudo -H systemctl restart uwsgi@${SEARX_UWSGI_APP%.*} +stop: sudo -H systemctl stop uwsgi@${SEARX_UWSGI_APP%.*} +disable: sudo -H systemctl disable uwsgi@${SEARX_UWSGI_APP%.*} +EOF + ;; + fedora-*) cat <<EOF +# systemd --> /usr/lib/systemd/system/uwsgi.service +# The unit file starts uWSGI in emperor mode (/etc/uwsgi.ini), see +# - https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html + +create: ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP} +restart: sudo -H touch ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP} +disable: sudo -H rm ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP} +EOF + ;; + esac + echo -e ".. END searx uwsgi-description $DIST_NAME" + + echo -e "\n.. START searx uwsgi-appini $DIST_NAME" + eval "echo \"$(< "${TEMPLATES}/${uWSGI_APPS_AVAILABLE}/${SEARX_UWSGI_APP}")\"" + echo -e "\n.. END searx uwsgi-appini $DIST_NAME" + + ) + done + +} + +# ---------------------------------------------------------------------------- +main "$@" +# ---------------------------------------------------------------------------- diff --git a/utils/site-python/sphinx_build_tools.py b/utils/site-python/sphinx_build_tools.py new file mode 100644 index 000000000..b9ebdeacc --- /dev/null +++ b/utils/site-python/sphinx_build_tools.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8; mode: python -*- +"""Implement some sphinx-build tools. + +""" + +import os +import sys +from sphinx.util.pycompat import execfile_ + +# ------------------------------------------------------------------------------ +def load_sphinx_config(namespace): +# ------------------------------------------------------------------------------ + + u"""Load an additional configuration file into *namespace*. + + The name of the configuration file is taken from the environment + ``SPHINX_CONF``. The external configuration file extends (or overwrites) the + configuration values from the origin ``conf.py``. With this you are able to + maintain *build themes*. To your docs/conf.py add:: + + from sphinx_build_tools import load_sphinx_config + ... + + # Since loadConfig overwrites settings from the global namespace, it has to be + # the last statement in the conf.py file + + load_sphinx_config(globals()) + + """ + + config_file = os.environ.get("SPHINX_CONF", None) + if (config_file is not None + and os.path.normpath(namespace["__file__"]) != os.path.normpath(config_file) ): + config_file = os.path.abspath(config_file) + + if os.path.isfile(config_file): + sys.stdout.write( + "load additional sphinx-config: %s\n" + % config_file) + config = namespace.copy() + config['__file__'] = config_file + execfile_(config_file, config) + del config['__file__'] + namespace.update(config) + else: + sys.stderr.write( + "WARNING: additional sphinx-config not found: %s\n" + % config_file) diff --git a/utils/templates/etc/apache2 b/utils/templates/etc/apache2 new file mode 120000 index 000000000..558a90717 --- /dev/null +++ b/utils/templates/etc/apache2 @@ -0,0 +1 @@ +httpd
\ No newline at end of file diff --git a/utils/templates/etc/filtron/rules.json b/utils/templates/etc/filtron/rules.json new file mode 100644 index 000000000..fff70fa8f --- /dev/null +++ b/utils/templates/etc/filtron/rules.json @@ -0,0 +1,129 @@ +[ + { + "name": "roboagent limit", + "filters": [ + "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client|Ruby|UniversalFeedParser)" + ], + "limit": 0, + "stop": true, + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "botlimit", + "filters": [ + "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)" + ], + "limit": 0, + "stop": true, + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "suspiciously frequent IP", + "filters": [], + "interval": 600, + "limit": 30, + "aggregations": [ + "Header:X-Forwarded-For" + ], + "actions":[ + {"name":"log"} + ] + }, + { + "name": "search request", + "filters": [ + "Param:q", + "Path=^(/|/search)$" + ], + "interval": 61, + "limit": 999, + "subrules": [ + { + "name": "missing Accept-Language", + "filters": ["!Header:Accept-Language"], + "limit": 0, + "stop": true, + "actions": [ + {"name":"log"}, + {"name": "block", + "params": {"message": "Rate limit exceeded"}} + ] + }, + { + "name": "suspiciously Connection=close header", + "filters": ["Header:Connection=close"], + "limit": 0, + "stop": true, + "actions": [ + {"name":"log"}, + {"name": "block", + "params": {"message": "Rate limit exceeded"}} + ] + }, + { + "name": "IP limit", + "interval": 61, + "limit": 9, + "stop": true, + "aggregations": [ + "Header:X-Forwarded-For" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "rss/json limit", + "filters": [ + "Param:format=(csv|json|rss)" + ], + "interval": 121, + "limit": 2, + "stop": true, + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "useragent limit", + "interval": 61, + "limit": 199, + "aggregations": [ + "Header:User-Agent" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + } + ] + } +] diff --git a/utils/templates/etc/httpd/sites-available/morty.conf b/utils/templates/etc/httpd/sites-available/morty.conf new file mode 100644 index 000000000..daeb3635a --- /dev/null +++ b/utils/templates/etc/httpd/sites-available/morty.conf @@ -0,0 +1,28 @@ +# -*- coding: utf-8; mode: apache -*- + +LoadModule headers_module ${APACHE_MODULES}/mod_headers.so +LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so +LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so +#LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so + +# SetEnvIf Request_URI "${PUBLIC_URL_PATH_MORTY}" dontlog +# CustomLog /dev/null combined env=dontlog + +<Location ${PUBLIC_URL_PATH_MORTY} > + + <IfModule mod_security2.c> + SecRuleEngine Off + </IfModule> + + Require all granted + + Order deny,allow + Deny from all + #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + ProxyPreserveHost On + ProxyPass http://${MORTY_LISTEN} + RequestHeader set X-Script-Name ${PUBLIC_URL_PATH_MORTY} + +</Location> diff --git a/utils/templates/etc/httpd/sites-available/searx.conf:filtron b/utils/templates/etc/httpd/sites-available/searx.conf:filtron new file mode 100644 index 000000000..379d47e24 --- /dev/null +++ b/utils/templates/etc/httpd/sites-available/searx.conf:filtron @@ -0,0 +1,33 @@ +# -*- coding: utf-8; mode: apache -*- + +LoadModule headers_module ${APACHE_MODULES}/mod_headers.so +LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so +LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so +#LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so + +# SetEnvIf Request_URI "${FILTRON_URL_PATH}" dontlog +# CustomLog /dev/null combined env=dontlog + +# SecRuleRemoveById 981054 +# SecRuleRemoveById 981059 +# SecRuleRemoveById 981060 +# SecRuleRemoveById 950907 + +<Location ${FILTRON_URL_PATH} > + + <IfModule mod_security2.c> + SecRuleEngine Off + </IfModule> + + Require all granted + + Order deny,allow + Deny from all + #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + ProxyPreserveHost On + ProxyPass http://${FILTRON_LISTEN} + RequestHeader set X-Script-Name ${FILTRON_URL_PATH} + +</Location> diff --git a/utils/templates/etc/httpd/sites-available/searx.conf:uwsgi b/utils/templates/etc/httpd/sites-available/searx.conf:uwsgi new file mode 100644 index 000000000..ef702de3a --- /dev/null +++ b/utils/templates/etc/httpd/sites-available/searx.conf:uwsgi @@ -0,0 +1,27 @@ +# -*- coding: utf-8; mode: apache -*- + +LoadModule headers_module ${APACHE_MODULES}/mod_headers.so +LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so +LoadModule proxy_uwsgi_module ${APACHE_MODULES}/mod_proxy_uwsgi.so +# LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so + +# SetEnvIf Request_URI "${SEARX_URL_PATH}" dontlog +# CustomLog /dev/null combined env=dontlog + +<Location ${SEARX_URL_PATH}> + + <IfModule mod_security2.c> + SecRuleEngine Off + </IfModule> + + Require all granted + + Order deny,allow + Deny from all + # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + ProxyPreserveHost On + ProxyPass unix:${SEARX_UWSGI_SOCKET}|uwsgi://uwsgi-uds-searx/ + +</Location> diff --git a/utils/templates/etc/nginx/default.apps-available/morty.conf b/utils/templates/etc/nginx/default.apps-available/morty.conf new file mode 100644 index 000000000..e7ffa27e2 --- /dev/null +++ b/utils/templates/etc/nginx/default.apps-available/morty.conf @@ -0,0 +1,11 @@ +# https://example.org/morty + +location /morty { + proxy_pass http://127.0.0.1:3000/; + + proxy_set_header Host \$http_host; + proxy_set_header Connection \$http_connection; + proxy_set_header X-Real-IP \$remote_addr; + proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; + proxy_set_header X-Scheme \$scheme; +} diff --git a/utils/templates/etc/nginx/default.apps-available/searx.conf:filtron b/utils/templates/etc/nginx/default.apps-available/searx.conf:filtron new file mode 100644 index 000000000..d3137e42d --- /dev/null +++ b/utils/templates/etc/nginx/default.apps-available/searx.conf:filtron @@ -0,0 +1,16 @@ +# https://example.org/searx + +location ${SEARX_URL_PATH} { + proxy_pass http://127.0.0.1:4004/; + + proxy_set_header Host \$http_host; + proxy_set_header Connection \$http_connection; + proxy_set_header X-Real-IP \$remote_addr; + proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; + proxy_set_header X-Scheme \$scheme; + proxy_set_header X-Script-Name ${SEARX_URL_PATH}; +} + +location ${SEARX_URL_PATH}/static { + alias ${SEARX_SRC}/searx/static; +} diff --git a/utils/templates/etc/uwsgi/apps-archlinux/searx.ini b/utils/templates/etc/uwsgi/apps-archlinux/searx.ini new file mode 100644 index 000000000..51f659d0f --- /dev/null +++ b/utils/templates/etc/uwsgi/apps-archlinux/searx.ini @@ -0,0 +1,80 @@ +[uwsgi] + +# uWSGI core +# ---------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core + +# Who will run the code +uid = ${SERVICE_USER} +gid = ${SERVICE_GROUP} + +# chdir to specified directory before apps loading +chdir = ${SEARX_SRC}/searx + +# searx configuration (settings.yml) +env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH} + +# disable logging for privacy +logger = systemd +disable-logging = true + +# The right granted on the created socket +chmod-socket = 666 + +# Plugin to use and interpretor config +single-interpreter = true + +# enable master process +master = true + +# load apps in each worker instead of the master +lazy-apps = true + +# load uWSGI plugins +plugin = python + +# By default the Python plugin does not initialize the GIL. This means your +# app-generated threads will not run. If you need threads, remember to enable +# them with enable-threads. Running uWSGI in multithreading mode (with the +# threads options) will automatically enable threading support. This *strange* +# default behaviour is for performance reasons. +enable-threads = true + + +# plugin: python +# -------------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python + +# load a WSGI module +module = searx.webapp + +# set PYTHONHOME/virtualenv +virtualenv = ${SEARX_PYENV} + +# add directory (or glob) to pythonpath +pythonpath = ${SEARX_SRC} + + +# speak to upstream +# ----------------- +# +# Activate the 'http' configuration for filtron or activate the 'socket' +# configuration if you setup your HTTP server to use uWSGI protocol via sockets. + +# using IP: +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http +# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html + +http = ${SEARX_INTERNAL_HTTP} + +# using unix-sockets: +# +# On some distributions you need to create the app folder for the sockets:: +# +# mkdir -p /run/uwsgi/app/searx +# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx +# +# socket = /run/uwsgi/app/searx/socket
\ No newline at end of file diff --git a/utils/templates/etc/uwsgi/apps-archlinux/searx.ini:socket b/utils/templates/etc/uwsgi/apps-archlinux/searx.ini:socket new file mode 100644 index 000000000..eeabb3715 --- /dev/null +++ b/utils/templates/etc/uwsgi/apps-archlinux/searx.ini:socket @@ -0,0 +1,80 @@ +[uwsgi] + +# uWSGI core +# ---------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core + +# Who will run the code +uid = ${SERVICE_USER} +gid = ${SERVICE_GROUP} + +# chdir to specified directory before apps loading +chdir = ${SEARX_SRC}/searx + +# searx configuration (settings.yml) +env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH} + +# disable logging for privacy +logger = systemd +disable-logging = true + +# The right granted on the created socket +chmod-socket = 666 + +# Plugin to use and interpretor config +single-interpreter = true + +# enable master process +master = true + +# load apps in each worker instead of the master +lazy-apps = true + +# load uWSGI plugins +plugin = python + +# By default the Python plugin does not initialize the GIL. This means your +# app-generated threads will not run. If you need threads, remember to enable +# them with enable-threads. Running uWSGI in multithreading mode (with the +# threads options) will automatically enable threading support. This *strange* +# default behaviour is for performance reasons. +enable-threads = true + + +# plugin: python +# -------------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python + +# load a WSGI module +module = searx.webapp + +# set PYTHONHOME/virtualenv +virtualenv = ${SEARX_PYENV} + +# add directory (or glob) to pythonpath +pythonpath = ${SEARX_SRC} + + +# speak to upstream +# ----------------- +# +# Activate the 'http' configuration for filtron or activate the 'socket' +# configuration if you setup your HTTP server to use uWSGI protocol via sockets. + +# using IP: +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http +# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html + +# http = ${SEARX_INTERNAL_HTTP} + +# using unix-sockets: +# +# On some distributions you need to create the app folder for the sockets:: +# +# mkdir -p /run/uwsgi/app/searx +# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx +# +socket = /run/uwsgi/app/searx/socket
\ No newline at end of file diff --git a/utils/templates/etc/uwsgi/apps-available/searx.ini b/utils/templates/etc/uwsgi/apps-available/searx.ini new file mode 100644 index 000000000..9785d7cd1 --- /dev/null +++ b/utils/templates/etc/uwsgi/apps-available/searx.ini @@ -0,0 +1,79 @@ +[uwsgi] + +# uWSGI core +# ---------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core + +# Who will run the code +uid = ${SERVICE_USER} +gid = ${SERVICE_GROUP} + +# chdir to specified directory before apps loading +chdir = ${SEARX_SRC}/searx + +# searx configuration (settings.yml) +env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH} + +# disable logging for privacy +disable-logging = true + +# The right granted on the created socket +chmod-socket = 666 + +# Plugin to use and interpretor config +single-interpreter = true + +# enable master process +master = true + +# load apps in each worker instead of the master +lazy-apps = true + +# load uWSGI plugins +plugin = python3,http + +# By default the Python plugin does not initialize the GIL. This means your +# app-generated threads will not run. If you need threads, remember to enable +# them with enable-threads. Running uWSGI in multithreading mode (with the +# threads options) will automatically enable threading support. This *strange* +# default behaviour is for performance reasons. +enable-threads = true + + +# plugin: python +# -------------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python + +# load a WSGI module +module = searx.webapp + +# set PYTHONHOME/virtualenv +virtualenv = ${SEARX_PYENV} + +# add directory (or glob) to pythonpath +pythonpath = ${SEARX_SRC} + + +# speak to upstream +# ----------------- +# +# Activate the 'http' configuration for filtron or activate the 'socket' +# configuration if you setup your HTTP server to use uWSGI protocol via sockets. + +# using IP: +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http +# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html + +http = ${SEARX_INTERNAL_HTTP} + +# using unix-sockets: +# +# On some distributions you need to create the app folder for the sockets:: +# +# mkdir -p /run/uwsgi/app/searx +# chmod -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx +# +# socket = /run/uwsgi/app/searx/socket
\ No newline at end of file diff --git a/utils/templates/etc/uwsgi/apps-available/searx.ini:socket b/utils/templates/etc/uwsgi/apps-available/searx.ini:socket new file mode 100644 index 000000000..88436e5eb --- /dev/null +++ b/utils/templates/etc/uwsgi/apps-available/searx.ini:socket @@ -0,0 +1,79 @@ +[uwsgi] + +# uWSGI core +# ---------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core + +# Who will run the code +uid = ${SERVICE_USER} +gid = ${SERVICE_GROUP} + +# chdir to specified directory before apps loading +chdir = ${SEARX_SRC}/searx + +# searx configuration (settings.yml) +env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH} + +# disable logging for privacy +disable-logging = true + +# The right granted on the created socket +chmod-socket = 666 + +# Plugin to use and interpretor config +single-interpreter = true + +# enable master process +master = true + +# load apps in each worker instead of the master +lazy-apps = true + +# load uWSGI plugins +plugin = python3,http + +# By default the Python plugin does not initialize the GIL. This means your +# app-generated threads will not run. If you need threads, remember to enable +# them with enable-threads. Running uWSGI in multithreading mode (with the +# threads options) will automatically enable threading support. This *strange* +# default behaviour is for performance reasons. +enable-threads = true + + +# plugin: python +# -------------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python + +# load a WSGI module +module = searx.webapp + +# set PYTHONHOME/virtualenv +virtualenv = ${SEARX_PYENV} + +# add directory (or glob) to pythonpath +pythonpath = ${SEARX_SRC} + + +# speak to upstream +# ----------------- +# +# Activate the 'http' configuration for filtron or activate the 'socket' +# configuration if you setup your HTTP server to use uWSGI protocol via sockets. + +# using IP: +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http +# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html + +# http = ${SEARX_INTERNAL_HTTP} + +# using unix-sockets: +# +# On some distributions you need to create the app folder for the sockets:: +# +# mkdir -p /run/uwsgi/app/searx +# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx +# +socket = /run/uwsgi/app/searx/socket
\ No newline at end of file diff --git a/utils/templates/lib/systemd/system/filtron.service b/utils/templates/lib/systemd/system/filtron.service new file mode 100644 index 000000000..3b0c6edcc --- /dev/null +++ b/utils/templates/lib/systemd/system/filtron.service @@ -0,0 +1,29 @@ +[Unit] + +Description=${SERVICE_NAME} +After=syslog.target +After=network.target + +[Service] + +Type=simple +User=${SERVICE_USER} +Group=${SERVICE_GROUP} +WorkingDirectory=${SERVICE_HOME} +ExecStart=${SERVICE_HOME}/go-apps/bin/filtron -api '${FILTRON_API}' -listen '${FILTRON_LISTEN}' -rules '${FILTRON_RULES}' -target '${FILTRON_TARGET}' + +Restart=always +Environment=USER=${SERVICE_USER} HOME=${SERVICE_HOME} + +# Some distributions may not support these hardening directives. If you cannot +# start the service due to an unknown option, comment out the ones not supported +# by your version of systemd. + +ProtectSystem=full +PrivateDevices=yes +PrivateTmp=yes +NoNewPrivileges=true + +[Install] + +WantedBy=multi-user.target diff --git a/utils/templates/lib/systemd/system/morty.service b/utils/templates/lib/systemd/system/morty.service new file mode 100644 index 000000000..25b676b51 --- /dev/null +++ b/utils/templates/lib/systemd/system/morty.service @@ -0,0 +1,29 @@ +[Unit] + +Description=${SERVICE_NAME} +After=syslog.target +After=network.target + +[Service] + +Type=simple +User=${SERVICE_USER} +Group=${SERVICE_GROUP} +WorkingDirectory=${SERVICE_HOME} +ExecStart=${SERVICE_HOME}/go-apps/bin/morty -key '${MORTY_KEY}' -listen '${MORTY_LISTEN}' -timeout ${MORTY_TIMEOUT} + +Restart=always +Environment=USER=${SERVICE_USER} HOME=${SERVICE_HOME} DEBUG=${SERVICE_ENV_DEBUG} + +# Some distributions may not support these hardening directives. If you cannot +# start the service due to an unknown option, comment out the ones not supported +# by your version of systemd. + +ProtectSystem=full +PrivateDevices=yes +PrivateTmp=yes +NoNewPrivileges=true + +[Install] + +WantedBy=multi-user.target |