diff options
43 files changed, 3901 insertions, 371 deletions
diff --git a/.config.mk b/.config.mk new file mode 100644 index 000000000..3349a7602 --- /dev/null +++ b/.config.mk @@ -0,0 +1,13 @@ +# -*- coding: utf-8; mode: makefile-gmake -*- +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# This environment is used by Makefile targets. If you not maintain your own +# searx brand, you normally not need to change the defaults (except SEARX_URL). +# Compare your settings here with file .config.sh used by the toolboxing in +# utils. + +export SEARX_URL=https://searx.me + +export GIT_URL=https://github.com/asciimoo/searx +export DOCS_URL=https://asciimoo.github.io/searx + diff --git a/.config.sh b/.config.sh new file mode 100644 index 000000000..0fdc0ca12 --- /dev/null +++ b/.config.sh @@ -0,0 +1,56 @@ +# -*- coding: utf-8; mode: sh -*- +# SPDX-License-Identifier: AGPL-3.0-or-later +# shellcheck shell=bash disable=SC2034 +# +# This environment is used by ./utils scripts like filtron.sh or searx.sh. The +# default values are *most flexible* and *best maintained*, you normally not +# need to change the defaults (except PUBLIC_URL). +# +# Before you change any value here you have to uninstall any previous +# installation. Further is it recommended to backup your changes simply by +# adding them to you local brand (git branch):: +# +# git add .config +# +# Compare your settings here with file .config.mk used by the Makefile targets. + +# The public URL of the searx instance: PUBLIC_URL="https://mydomain.xy/searx" +# The default is taken from the Makefile environment (SEARX_URL). +PUBLIC_URL="${SEARX_URL:-http://$(uname -n)/searx}" +PUBLIC_HOST="${PUBLIC_HOST:-$(echo "$PUBLIC_URL" | sed -e 's/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/')}" + +# searx.sh +# --------- + +SEARX_INTERNAL_URL="${SEARX_INTERNAL_URL:-127.0.0.1:8888}" + +# Only change, if you maintain a searx brand in your searx fork. The default is +# taken from the Makefile environment (DOCS_URL, GIT_URL). +SEARX_DOCS_URL="${DOCS_URL:-https://asciimoo.github.io/searx}" +SEARX_GIT_URL="${GIT_URL:-https://github.com/asciimoo/searx.git}" +SEARX_GIT_BRANCH="${SEARX_GIT_BRANCH:-master}" + +# filtron.sh +# ---------- + +FILTRON_API="${FILTRON_API:-127.0.0.1:4005}" +FILTRON_LISTEN="${FILTRON_LISTEN:-127.0.0.1:4004}" +FILTRON_TARGET="${FILTRON_TARGET:-127.0.0.1:8888}" + +# morty.sh +# -------- + +# morty listen address +MORTY_LISTEN="${MORTY_LISTEN:-127.0.0.1:3000}" + +# system services +# --------------- + +# **experimental**: Set SERVICE_USER to run all services by one account, but be +# aware that removing discrete components might conflict! +# +# SERVICE_USER=searx + +# Common $HOME folder of the service accounts +SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}" + diff --git a/.gitignore b/.gitignore index 069dfd35b..3c998afae 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # to sync with .dockerignore .coverage coverage/ +cache/ .installed.cfg engines.cfg env @@ -1,8 +1,6 @@ # -*- coding: utf-8; mode: makefile-gmake -*- -export GIT_URL=https://github.com/asciimoo/searx -export SEARX_URL=https://searx.me -export DOCS_URL=https://asciimoo.github.io/searx +include ./.config.mk PYOBJECTS = searx DOC = docs @@ -70,12 +68,20 @@ $(GH_PAGES):: # test # ---- -PHONY += test test.pylint test.pep8 test.unit test.robot +PHONY += test test.sh test.pylint test.pep8 test.unit test.robot # TODO: balance linting with pylint -test: test.pep8 test.unit test.robot +test: test.pep8 test.unit test.sh test.robot - make pylint +test.sh: + shellcheck -x utils/lxc.sh + shellcheck -x utils/lib.sh + shellcheck -x utils/filtron.sh + shellcheck -x utils/searx.sh + shellcheck -x utils/morty.sh + shellcheck -x .config.sh + test.pep8: pyenvinstall $(PY_ENV_ACT); ./manage.sh pep8_check diff --git a/docs/_themes/searx/static/searx.css b/docs/_themes/searx/static/searx.css index d6a664f0f..ae742a4e0 100644 --- a/docs/_themes/searx/static/searx.css +++ b/docs/_themes/searx/static/searx.css @@ -33,7 +33,7 @@ p.sidebar-title, .sidebar p { /* admonitions */ -div.admonition, div.topic { +div.admonition, div.topic, div.toctree-wrapper { background-color: #fafafa; margin: 8px 0px; padding: 1em; @@ -42,6 +42,16 @@ div.admonition, div.topic { border-right: none; border-bottom: none; border-left: 5pt solid #ccc; + list-style-type: disclosure-closed; +} + +div.toctree-wrapper p.caption { + font-weight: normal; + font-size: 24px; + margin: 0 0 10px 0; + padding: 0; + line-height: 1; + display: inline; } p.admonition-title:after { diff --git a/docs/admin/arch_public.dot b/docs/admin/arch_public.dot index a46c96de3..5018225c3 100644 --- a/docs/admin/arch_public.dot +++ b/docs/admin/arch_public.dot @@ -4,11 +4,11 @@ digraph G { edge [fontname="Sans"]; browser [label="Browser", shape=Mdiamond]; - rp [label="Reverse Proxy", href="url to configure reverse proxy"]; - filtron [label="Filtron", href="https://github.com/asciimoo/filtron"]; - morty [label="Morty", href="https://github.com/asciimoo/morty"]; + rp [label="Reverse Proxy", href="https://asciimoo.github.io/searx/utils/filtron.sh.html#public-reverse-proxy"]; + filtron [label="Filtron", href="https://asciimoo.github.io/searx/utils/filtron.sh.html"]; + morty [label="Morty", href="https://asciimoo.github.io/searx/utils/morty.sh.html"]; static [label="Static files", href="url to configure static files"]; - uwsgi [label="uwsgi", href="url to configure uwsgi"] + uwsgi [label="uwsgi", href="https://asciimoo.github.io/searx/utils/searx.sh.html"] searx1 [label="Searx #1"]; searx2 [label="Searx #2"]; searx3 [label="Searx #3"]; diff --git a/docs/admin/architecture.rst b/docs/admin/architecture.rst index 7064a294b..fbc3bf483 100644 --- a/docs/admin/architecture.rst +++ b/docs/admin/architecture.rst @@ -4,17 +4,19 @@ Architecture ============ -.. sidebar:: Needs work! +.. sidebar:: Tooling box - This article needs some work / Searx is a collaborative effort. If you have - any contribution, feel welcome to send us your :pull:`PR <../pulls>`, see - :ref:`how to contribute`. + - :ref:`searx & uwsgi <searx.sh>` + - :ref:`filtron <filtron.sh>` + - :ref:`reverse proxy` + - :ref:`morty <morty.sh>` Herein you will find some hints and suggestions about typical architectures of searx infrastructures. We start with a contribution from :pull:`@dalf <1776#issuecomment-567917320>`. -It shows a *reference* setup for public searx instances. +It shows a *reference* setup for public searx instances which can build up and +maintained by the scripts from our :ref:`toolboxing`. .. _arch public: diff --git a/docs/admin/buildhosts.rst b/docs/admin/buildhosts.rst index 5260da033..27784764f 100644 --- a/docs/admin/buildhosts.rst +++ b/docs/admin/buildhosts.rst @@ -4,13 +4,18 @@ Buildhosts ========== +To get best results from build, its recommend to install additional packages +on build hosts. + .. sidebar:: This article needs some work If you have any contribution send us your :pull:`PR <../pulls>`, see :ref:`how to contribute`. -To get best results from build, its recommend to install additional packages -on build hosts. +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry .. _docs build: @@ -35,8 +40,17 @@ processing additional packages are needed. The XeTeX_ needed not only for PDF creation, its also needed for :ref:`math` when HTML output is build. To be able to do :ref:`sphinx:math-support` without CDNs, the math are rendered -as images (``sphinx.ext.imgmath`` extension). If your docs build (``make -docs``) shows warnings like this:: +as images (``sphinx.ext.imgmath`` extension). + +Here is the extract from the :origin:`docs/conf.py` file, setting math renderer +to ``imgmath``: + +.. literalinclude:: ../conf.py + :language: python + :start-after: # sphinx.ext.imgmath setup + :end-before: # sphinx.ext.imgmath setup END + +If your docs build (``make docs``) shows warnings like this:: WARNING: dot(1) not found, for better output quality install \ graphviz from http://www.graphviz.org @@ -47,8 +61,6 @@ docs``) shows warnings like this:: you need to install additional packages on your build host, to get better HTML output. -.. _system requirements: - .. tabs:: .. group-tab:: Ubuntu / debian @@ -94,10 +106,35 @@ For PDF output you also need: texlive-collection-fontsrecommended texlive-collection-latex \ dejavu-sans-fonts dejavu-serif-fonts dejavu-sans-mono-fonts -.. _system requirements END: +.. _sh lint: -.. literalinclude:: ../conf.py - :language: python - :start-after: # sphinx.ext.imgmath setup - :end-before: # sphinx.ext.imgmath setup END +Lint shell scripts +================== + +.. _ShellCheck: https://github.com/koalaman/shellcheck + +To lint shell scripts, we use ShellCheck_ - A shell script static analysis tool. + +.. SNIP sh lint requirements + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code-block:: sh + + $ sudo apt install shellcheck + + .. group-tab:: Arch Linux + + .. code-block:: sh + + $ sudo pacman -S shellcheck + + .. group-tab:: Fedora / RHEL + + .. code-block:: sh + + $ sudo dnf install ShellCheck +.. SNAP sh lint requirements diff --git a/docs/admin/filtron.rst b/docs/admin/filtron.rst index 07dcb9bc5..2bc663411 100644 --- a/docs/admin/filtron.rst +++ b/docs/admin/filtron.rst @@ -1,18 +1,48 @@ + +.. _searx_filtron: + ========================== How to protect an instance ========================== +.. sidebar:: further reading + + - :ref:`filtron.sh` + +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + +.. _filtron: https://github.com/asciimoo/filtron + Searx depens on external search services. To avoid the abuse of these services it is advised to limit the number of requests processed by searx. -An application firewall, ``filtron`` solves exactly this problem. Information -on how to install it can be found at the `project page of filtron -<https://github.com/asciimoo/filtron>`__. +An application firewall, filtron_ solves exactly this problem. Filtron is just +a middleware between your web server (nginx, apache, ...) and searx, we describe +such infratructures in chapter: :ref:`architecture`. + + +filtron & go +============ + +.. _Go: https://golang.org/ +.. _filtron README: https://github.com/asciimoo/filtron/blob/master/README.md + +Filtron needs Go_ installed. If Go_ is preinstalled, filtron_ is simply +installed by ``go get`` package management (see `filtron README`_). If you use +filtron as middleware, a more isolated setup is recommended. To simplify such +an installation and the maintenance of, use our script :ref:`filtron.sh`. Sample configuration of filtron =============================== +.. sidebar:: Tooling box + + - :origin:`/etc/filtron/rules.json <utils/templates/etc/filtron/rules.json>` + An example configuration can be find below. This configuration limits the access of: @@ -24,100 +54,100 @@ of: .. code:: json - [{ - "name":"search request", - "filters":[ - "Param:q", - "Path=^(/|/search)$" - ], - "interval":"<time-interval-in-sec (int)>", - "limit":"<max-request-number-in-interval (int)>", - "subrules":[ - { - "name":"roboagent limit", - "interval":"<time-interval-in-sec (int)>", - "limit":"<max-request-number-in-interval (int)>", - "filters":[ - "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)" - ], - "actions":[ - { - "name":"block", - "params":{ - "message":"Rate limit exceeded" - } - } - ] - }, - { - "name":"botlimit", - "limit":0, - "stop":true, - "filters":[ - "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)" - ], - "actions":[ - { - "name":"block", - "params":{ - "message":"Rate limit exceeded" - } - } - ] - }, - { - "name":"IP limit", - "interval":"<time-interval-in-sec (int)>", - "limit":"<max-request-number-in-interval (int)>", - "stop":true, - "aggregations":[ - "Header:X-Forwarded-For" - ], - "actions":[ - { - "name":"block", - "params":{ - "message":"Rate limit exceeded" - } - } - ] - }, - { - "name":"rss/json limit", - "interval":"<time-interval-in-sec (int)>", - "limit":"<max-request-number-in-interval (int)>", - "stop":true, - "filters":[ - "Param:format=(csv|json|rss)" - ], - "actions":[ - { - "name":"block", - "params":{ - "message":"Rate limit exceeded" - } - } - ] - }, - { - "name":"useragent limit", - "interval":"<time-interval-in-sec (int)>", - "limit":"<max-request-number-in-interval (int)>", - "aggregations":[ - "Header:User-Agent" - ], - "actions":[ - { - "name":"block", - "params":{ - "message":"Rate limit exceeded" - } - } - ] - } - ] - }] - + [ + { "name": "search request", + "filters": [ + "Param:q", + "Path=^(/|/search)$" + ], + "interval": "<time-interval-in-sec (int)>", + "limit": "<max-request-number-in-interval (int)>", + "subrules": [ + { + "name": "roboagent limit", + "interval": "<time-interval-in-sec (int)>", + "limit": "<max-request-number-in-interval (int)>", + "filters": [ + "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "botlimit", + "limit": 0, + "stop": true, + "filters": [ + "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "IP limit", + "interval": "<time-interval-in-sec (int)>", + "limit": "<max-request-number-in-interval (int)>", + "stop": true, + "aggregations": [ + "Header:X-Forwarded-For" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "rss/json limit", + "interval": "<time-interval-in-sec (int)>", + "limit": "<max-request-number-in-interval (int)>", + "stop": true, + "filters": [ + "Param:format=(csv|json|rss)" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "useragent limit", + "interval": "<time-interval-in-sec (int)>", + "limit": "<max-request-number-in-interval (int)>", + "aggregations": [ + "Header:User-Agent" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + } + ] + } + ] Route request through filtron diff --git a/docs/admin/index.rst b/docs/admin/index.rst index b3c7f5119..4190bf24b 100644 --- a/docs/admin/index.rst +++ b/docs/admin/index.rst @@ -3,9 +3,12 @@ Administrator documentation =========================== .. toctree:: - :maxdepth: 1 + :maxdepth: 2 + :caption: Contents installation + installation-nginx + installation-apache settings api architecture diff --git a/docs/admin/installation-apache.rst b/docs/admin/installation-apache.rst new file mode 100644 index 000000000..7f92e47d5 --- /dev/null +++ b/docs/admin/installation-apache.rst @@ -0,0 +1,94 @@ +.. _installation apache: + +=================== +Install with apache +=================== + +.. sidebar:: public HTTP servers + + On public searx instances use an application firewall (:ref:`filtron + <filtron.sh>`). + +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + +Add wsgi mod +============ + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code:: sh + + sudo -H apt-get install libapache2-mod-uwsgi + sudo -H a2enmod uwsgi + +Add this configuration in the file ``/etc/apache2/apache2.conf``. To limit +acces to your intranet replace ``Allow from all`` directive and replace +``192.168.0.0/16`` with your subnet IP/class. + +.. _inranet apache site: + +Note that if your instance of searx is not at the root, you should change +``<Location />`` by the location of your instance, like ``<Location /searx>``: + +.. code:: apache + + # CustomLog /dev/null combined + + <IfModule mod_uwsgi.c> + + <Location /> + + Options FollowSymLinks Indexes + SetHandler uwsgi-handler + uWSGISocket /run/uwsgi/app/searx/socket + + Order deny,allow + Deny from all + # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + </Location> + + </IfModule> + +Enable apache mod_uwsgi and restart apache: + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code:: sh + + a2enmod uwsgi + sudo -H systemctl restart apache2 + +disable logs +============ + +For better privacy you can disable Apache logs. Go back to +``/etc/apache2/apache2.conf`` :ref:`[example] <inranet apache site>` and above +``<Location />`` activate directive: + +.. code:: apache + + CustomLog /dev/null combined + +Restart apache: + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code:: sh + + sudo -H systemctl restart apache2 + +.. warning:: + + You can only disable logs for the whole (virtual) server not for a specific + path. diff --git a/docs/admin/installation-nginx.rst b/docs/admin/installation-nginx.rst new file mode 100644 index 000000000..37d3e7532 --- /dev/null +++ b/docs/admin/installation-nginx.rst @@ -0,0 +1,141 @@ +.. _installation nginx: + +================== +Install with nginx +================== + +.. sidebar:: public HTTP servers + + On public searx instances use an application firewall (:ref:`filtron + <filtron.sh>`). + +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + +If nginx is not installed (uwsgi will not work with the package +nginx-light): + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code:: sh + + sudo -H apt-get install nginx + +Hosted at ``/`` +=============== + +Create the configuration file ``/etc/nginx/sites-available/searx`` with this +content: + +.. code:: nginx + + server { + listen 80; + server_name searx.example.com; + root /usr/local/searx/searx; + + location /static { + } + + location / { + include uwsgi_params; + uwsgi_pass unix:/run/uwsgi/app/searx/socket; + } + } + +Create a symlink to sites-enabled: + +.. code:: sh + + sudo -H ln -s /etc/nginx/sites-available/searx /etc/nginx/sites-enabled/searx + +Restart service: + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code:: sh + + sudo -H systemctl restart nginx + sudo -H systemctl restart uwsgi + +from subdirectory URL (``/searx``) +================================== + +Add this configuration in the server config file +``/etc/nginx/sites-enabled/default``: + +.. code:: nginx + + location /searx/static { + alias /usr/local/searx/searx/static; + } + + location /searx { + uwsgi_param SCRIPT_NAME /searx; + include uwsgi_params; + uwsgi_pass unix:/run/uwsgi/app/searx/socket; + } + + +**OR** using reverse proxy (Please, note that reverse proxy advised to be used +in case of single-user or low-traffic instances.) + +.. code:: nginx + + location /searx/static { + alias /usr/local/searx/searx/static; + } + + location /searx { + proxy_pass http://127.0.0.1:8888; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Script-Name /searx; + proxy_buffering off; + } + +Enable ``base_url`` in ``searx/settings.yml`` + +.. code:: yaml + + base_url : http://your.domain.tld/searx/ + +Restart service: + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code:: sh + + sudo -H systemctl restart nginx + sudo -H systemctl restart uwsgi + + +disable logs +============ + +For better privacy you can disable nginx logs about searx. How to proceed: +below ``uwsgi_pass`` in ``/etc/nginx/sites-available/default`` add: + +.. code:: nginx + + access_log /dev/null; + error_log /dev/null; + +Restart service: + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code:: sh + + sudo -H systemctl restart nginx diff --git a/docs/admin/installation.rst b/docs/admin/installation.rst index 15800fc01..a4c7eb8b3 100644 --- a/docs/admin/installation.rst +++ b/docs/admin/installation.rst @@ -4,44 +4,64 @@ Installation ============ -.. contents:: - :depth: 3 +.. sidebar:: Searx server setup + + - :ref:`installation nginx` + - :ref:`installation apache` + + If you do not have any special preferences, it is recommend to use + :ref:`searx.sh`. + +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + +.. _installation basic: Basic installation ================== -Step by step installation for Debian/Ubuntu with virtualenv. For Ubuntu, be sure -to have enable universe repository. +Step by step installation with virtualenv. For Ubuntu, be sure to have enable +universe repository. Install packages: -.. code:: sh +.. tabs:: + + .. group-tab:: Ubuntu / debian - $ sudo -H apt-get install \ - git build-essential libxslt-dev \ - python-dev python-virtualenv python-babel \ - zlib1g-dev libffi-dev libssl-dev + .. code-block:: sh + + $ sudo -H apt-get install -m \ + git build-essential + libxslt-dev python3-dev python3-babel python3-venv \ + zlib1g-dev libffi-dev libssl-dev Install searx: .. code:: sh - cd /usr/local - sudo -H git clone https://github.com/asciimoo/searx.git - sudo -H useradd searx -d /usr/local/searx + sudo -H useradd searx --system --disabled-password -d /usr/local/searx + sudo -H usermod -a -G shadow searx + cd /usr/local/searx + sudo -H git clone https://github.com/asciimoo/searx.git searx-src sudo -H chown searx:searx -R /usr/local/searx -Install dependencies in a virtualenv: +Install virtualenv: .. code:: sh - cd /usr/local/searx sudo -H -u searx -i + (searx)$ python3 -m venv searx-pyenv + (searx)$ echo 'source ~/searx-pyenv/bin/activate' > ~/.profile + +Exit the searx bash and restart a new to install the searx dependencies: .. code:: sh - (searx)$ virtualenv searx-ve - (searx)$ . ./searx-ve/bin/activate + sudo -H -u searx -i + (searx)$ cd searx-src (searx)$ ./manage.sh update_packages Configuration @@ -49,7 +69,9 @@ Configuration .. code:: sh - sed -i -e "s/ultrasecretkey/`openssl rand -hex 16`/g" searx/settings.yml + sudo -H -u searx -i + (searx)$ cd searx-src + (searx)$ sed -i -e "s/ultrasecretkey/`openssl rand -hex 16`/g" searx/settings.yml Edit searx/settings.yml if necessary. @@ -60,7 +82,9 @@ Start searx: .. code:: sh - python searx/webapp.py + sudo -H -u searx -i + (searx)$ cd searx-src + (searx)$ python3 searx/webapp.py Go to http://localhost:8888 @@ -70,254 +94,118 @@ If everything works fine, disable the debug option in settings.yml: sed -i -e "s/debug : True/debug : False/g" searx/settings.yml -At this point searx is not demonized ; uwsgi allows this. - -You can exit the virtualenv and the searx user bash (enter exit command -twice). +At this point searx is not demonized ; uwsgi allows this. You can exit the +virtualenv and the searx user bash (enter exit command twice). uwsgi ===== Install packages: -.. code:: sh - - sudo -H apt-get install \ - uwsgi uwsgi-plugin-python - -Create the configuration file ``/etc/uwsgi/apps-available/searx.ini`` with this -content: - -.. code:: ini - - [uwsgi] - # Who will run the code - uid = searx - gid = searx - - # disable logging for privacy - disable-logging = true - - # Number of workers (usually CPU count) - workers = 4 +.. tabs:: - # The right granted on the created socket - chmod-socket = 666 + .. group-tab:: Ubuntu / debian - # Plugin to use and interpretor config - single-interpreter = true - master = true - plugin = python - lazy-apps = true - enable-threads = true + .. code-block:: bash - # Module to import - module = searx.webapp + sudo -H apt-get install uwsgi uwsgi-plugin-python3 - # Support running the module from a webserver subdirectory. - route-run = fixpathinfo: - - # Virtualenv and python path - virtualenv = /usr/local/searx/searx-ve/ - pythonpath = /usr/local/searx/ - chdir = /usr/local/searx/searx/ - -Activate the uwsgi application and restart: - -.. code:: sh - - cd /etc/uwsgi/apps-enabled - ln -s ../apps-available/searx.ini - /etc/init.d/uwsgi restart - -Web server -========== - -with nginx ----------- - -If nginx is not installed (uwsgi will not work with the package -nginx-light): - -.. code:: sh - - sudo -H apt-get install nginx - -Hosted at / -~~~~~~~~~~~ - -Create the configuration file ``/etc/nginx/sites-available/searx`` with this +Create the configuration file ``/etc/uwsgi/apps-available/searx.ini`` with this content: -.. code:: nginx - - server { - listen 80; - server_name searx.example.com; - root /usr/local/searx/searx; - - location /static { - } - - location / { - include uwsgi_params; - uwsgi_pass unix:/run/uwsgi/app/searx/socket; - } - } - -Create a symlink to sites-enabled: - -.. code:: sh - - sudo -H ln -s /etc/nginx/sites-available/searx /etc/nginx/sites-enabled/searx - -Restart service: - -.. code:: sh - - sudo -H service nginx restart - sudo -H service uwsgi restart - -from subdirectory URL (/searx) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Add this configuration in the server config file -``/etc/nginx/sites-enabled/default``: - -.. code:: nginx - - location /searx/static { - alias /usr/local/searx/searx/static; - } - - location /searx { - uwsgi_param SCRIPT_NAME /searx; - include uwsgi_params; - uwsgi_pass unix:/run/uwsgi/app/searx/socket; - } - - -**OR** using reverse proxy (Please, note that reverse proxy advised to be used -in case of single-user or low-traffic instances.) - -.. code:: nginx - - location /searx/static { - alias /usr/local/searx/searx/static; - } - - location /searx { - proxy_pass http://127.0.0.1:8888; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Scheme $scheme; - proxy_set_header X-Script-Name /searx; - proxy_buffering off; - } - - -Enable ``base_url`` in ``searx/settings.yml`` - -.. code:: yaml - - base_url : http://your.domain.tld/searx/ +.. code:: ini -Restart service: + [uwsgi] -.. code:: sh + # uWSGI core + # ---------- + # + # https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core - sudo -H service nginx restart - sudo -H service uwsgi restart + # Who will run the code + uid = searx + gid = searx -disable logs -^^^^^^^^^^^^ + # chdir to specified directory before apps loading + chdir = /usr/local/searx/searx-src/searx -for better privacy you can disable nginx logs about searx. + # disable logging for privacy + disable-logging = true -how to proceed: below ``uwsgi_pass`` in ``/etc/nginx/sites-available/default`` -add: + # The right granted on the created socket + chmod-socket = 666 -.. code:: nginx + # Plugin to use and interpretor config + single-interpreter = true - access_log /dev/null; - error_log /dev/null; + # enable master process + master = true -Restart service: + # load apps in each worker instead of the master + lazy-apps = true -.. code:: sh + # load uWSGI plugins + plugin = python3,http - sudo -H service nginx restart + # By default the Python plugin does not initialize the GIL. This means your + # app-generated threads will not run. If you need threads, remember to enable + # them with enable-threads. Running uWSGI in multithreading mode (with the + # threads options) will automatically enable threading support. This *strange* + # default behaviour is for performance reasons. + enable-threads = true -with apache ------------ + # plugin: python + # -------------- + # + # https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python -Add wsgi mod: + # load a WSGI module + module = searx.webapp -.. code:: sh + # set PYTHONHOME/virtualenv + virtualenv = /usr/local/searx/searx-pyenv - sudo -H apt-get install libapache2-mod-uwsgi - sudo -H a2enmod uwsgi + # add directory (or glob) to pythonpath + pythonpath = /usr/local/searx/searx-src -Add this configuration in the file ``/etc/apache2/apache2.conf``: -.. code:: apache + # plugin http + # ----------- + # + # https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http - <Location /> - Options FollowSymLinks Indexes - SetHandler uwsgi-handler - uWSGISocket /run/uwsgi/app/searx/socket - </Location> + # Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html + http = 127.0.0.1:8888 -Note that if your instance of searx is not at the root, you should change -``<Location />`` by the location of your instance, like ``<Location /searx>``. - -Restart Apache: +Activate the uwsgi application and restart: .. code:: sh - sudo -H /etc/init.d/apache2 restart - -disable logs -~~~~~~~~~~~~ - -For better privacy you can disable Apache logs. - -.. warning:: - - You can only disable logs for the whole (virtual) server not for a specific - path. - -Go back to ``/etc/apache2/apache2.conf`` and above ``<Location />`` add: - -.. code:: apache - - CustomLog /dev/null combined - -Restart Apache: - -.. code:: sh + cd /etc/uwsgi/apps-enabled + ln -s ../apps-available/searx.ini + /etc/init.d/uwsgi restart - sudo -H /etc/init.d/apache2 restart How to update ============= .. code:: sh - cd /usr/local/searx sudo -H -u searx -i - -.. code:: sh - - (searx)$ . ./searx-ve/bin/activate (searx)$ git stash (searx)$ git pull origin master (searx)$ git stash apply (searx)$ ./manage.sh update_packages -.. code:: sh +Restart uwsgi: + +.. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code:: sh - sudo -H service uwsgi restart + sudo -H systemctl restart uwsgi Docker ====== diff --git a/docs/admin/morty.rst b/docs/admin/morty.rst index 7d7b34492..9af9b6ae9 100644 --- a/docs/admin/morty.rst +++ b/docs/admin/morty.rst @@ -1,3 +1,6 @@ + +.. _searx_morty: + ========================= How to setup result proxy ========================= diff --git a/docs/admin/settings.rst b/docs/admin/settings.rst index 0bfdcc6cb..59a406a28 100644 --- a/docs/admin/settings.rst +++ b/docs/admin/settings.rst @@ -4,11 +4,17 @@ ``settings.yml`` ================ +This page describe the options possibilities of the :origin:`searx/settings.yml` +file. + .. sidebar:: Further reading .. - :ref:`search API` -This page describe the options possibilities of the settings.yml file. +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry .. _settings global: diff --git a/docs/blog/index.rst b/docs/blog/index.rst index 52fa3f126..bc90f4502 100644 --- a/docs/blog/index.rst +++ b/docs/blog/index.rst @@ -3,7 +3,8 @@ Blog ==== .. toctree:: - :maxdepth: 1 + :maxdepth: 2 + :caption: Contents python3 admin diff --git a/docs/conf.py b/docs/conf.py index cd61b697e..5fbdb54bb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -5,6 +5,7 @@ from searx.version import VERSION_STRING from pallets_sphinx_themes import ProjectLink GIT_URL = os.environ.get("GIT_URL", "https://github.com/asciimoo/searx") +GIT_BRANCH =os.environ.get("GIT_BRANCH", "master") SEARX_URL = os.environ.get("SEARX_URL", "https://searx.me") DOCS_URL = os.environ.get("DOCS_URL", "https://asciimoo.github.io/searx/") @@ -35,7 +36,7 @@ extlinks['wiki'] = ('https://github.com/asciimoo/searx/wiki/%s', ' ') extlinks['pull'] = ('https://github.com/asciimoo/searx/pull/%s', 'PR ') # links to custom brand -extlinks['origin'] = (GIT_URL + '/blob/master/%s', 'git://') +extlinks['origin'] = (GIT_URL + '/blob/' + GIT_BRANCH + '/%s', 'git://') extlinks['patch'] = (GIT_URL + '/commit/%s', '#') extlinks['search'] = (SEARX_URL + '/%s', '#') extlinks['docs'] = (DOCS_URL + '/%s', 'docs: ') @@ -61,6 +62,7 @@ extensions = [ "pallets_sphinx_themes", "sphinx_issues", # https://github.com/sloria/sphinx-issues/blob/master/README.rst "sphinxcontrib.jinja", # https://github.com/tardyp/sphinx-jinja + "sphinxcontrib.programoutput", # https://github.com/NextThought/sphinxcontrib-programoutput 'linuxdoc.rstFlatTable', # Implementation of the 'flat-table' reST-directive. 'linuxdoc.kfigure', # Sphinx extension which implements scalable image handling. "sphinx_tabs.tabs", # https://github.com/djungelorm/sphinx-tabs diff --git a/docs/dev/contribution_guide.rst b/docs/dev/contribution_guide.rst index 459dfb448..f9e9569f9 100644 --- a/docs/dev/contribution_guide.rst +++ b/docs/dev/contribution_guide.rst @@ -4,6 +4,11 @@ How to contribute ================= +.. contents:: Contents + :depth: 2 + :local: + :backlinks: entry + Prime directives: Privacy, Hackability ====================================== diff --git a/docs/dev/index.rst b/docs/dev/index.rst index cb913a82b..ba0a25a9c 100644 --- a/docs/dev/index.rst +++ b/docs/dev/index.rst @@ -3,7 +3,8 @@ Developer documentation ======================= .. toctree:: - :maxdepth: 1 + :maxdepth: 2 + :caption: Contents quickstart contribution_guide diff --git a/docs/dev/makefile.rst b/docs/dev/makefile.rst index f5957001c..ed9942f4c 100644 --- a/docs/dev/makefile.rst +++ b/docs/dev/makefile.rst @@ -11,6 +11,8 @@ Makefile Targets Before looking deeper at the targets, first read about :ref:`makefile setup` and :ref:`make pyenv`. + To install system requirements follow :ref:`buildhosts`. + With the aim to simplify development cycles, started with :pull:`1756` a ``Makefile`` based boilerplate was added. If you are not familiar with Makefiles, we recommend to read gnu-make_ introduction. @@ -37,15 +39,17 @@ Calling the ``help`` target gives a first overview:: .. _makefile setup: -Setup -===== +Makefile setup +============== .. _git stash: https://git-scm.com/docs/git-stash -The main setup is done in the :origin:`Makefile`:: +The main setup is done in the :origin:`.config.sh` (read :ref:`toolboxing +setup`):: - export GIT_URL=https://github.com/asciimoo/searx export SEARX_URL=https://searx.me + + export GIT_URL=https://github.com/asciimoo/searx export DOCS_URL=https://asciimoo.github.io/searx .. sidebar:: fork & upstream @@ -170,7 +174,7 @@ e.g.: .. code:: sh - $ make test.pep8 test.unit + $ make test.pep8 test.unit test.sh . ./local/py3/bin/activate; ./manage.sh pep8_check [!] Running pep8 check . ./local/py3/bin/activate; ./manage.sh unit_tests diff --git a/docs/dev/reST.rst b/docs/dev/reST.rst index 4dc1279f0..7cf9b4be8 100644 --- a/docs/dev/reST.rst +++ b/docs/dev/reST.rst @@ -326,7 +326,7 @@ Literal blocks The simplest form of :duref:`literal-blocks` is a indented block introduced by two colons (``::``). For highlighting use :dudir:`highlight` or :ref:`reST code` directive. To include literals from external files use directive -:dudir:`literalinclude`. +:rst:dir:`literalinclude`. .. _reST literal: @@ -1312,9 +1312,8 @@ others are basic-tabs_ and code-tabs_. Below a *group-tab* example from .. literalinclude:: ../admin/buildhosts.rst :language: reST - :start-after: .. _system requirements: - :end-before: .. _system requirements END: - + :start-after: .. SNIP sh lint requirements + :end-before: .. SNAP sh lint requirements .. _math: diff --git a/docs/index.rst b/docs/index.rst index d9503fef6..c6f5eee96 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,7 +2,14 @@ Welcome to searx ================ -Search without being tracked. + *Search without being tracked.* + +Searx is a free internet metasearch engine which aggregates results from more +than 70 search services. Users are neither tracked nor profiled. Additionally, +searx can be used over Tor for online anonymity. + +Get started with searx by using one of the :wiki:`Searx-instances`. If you +don't trust anyone, you can set up your own, see :ref:`installation`. .. sidebar:: Features @@ -16,17 +23,12 @@ Search without being tracked. - Hosted by organizations, such as *La Quadrature du Net*, which promote digital rights -Searx is a free internet metasearch engine which aggregates results from more -than 70 search services. Users are neither tracked nor profiled. Additionally, -searx can be used over Tor for online anonymity. - -Get started with searx by using one of the :wiki:`Searx-instances`. If you -don't trust anyone, you can set up your own, see :ref:`installation`. - .. toctree:: :maxdepth: 2 + :caption: Contents user/index admin/index dev/index + utils/index blog/index diff --git a/docs/user/index.rst b/docs/user/index.rst index b13aca216..96d11bf8a 100644 --- a/docs/user/index.rst +++ b/docs/user/index.rst @@ -3,7 +3,8 @@ User documentation ================== .. toctree:: - :maxdepth: 1 + :maxdepth: 2 + :caption: Contents search_syntax own-instance diff --git a/docs/user/own-instance.rst b/docs/user/own-instance.rst index a2f736562..4876abe99 100644 --- a/docs/user/own-instance.rst +++ b/docs/user/own-instance.rst @@ -2,8 +2,10 @@ Why use a private instance? =========================== -"Is it worth to run my own instance?" is a common question among searx users. -Before answering this question, see what options a searx user has. + *"Is it worth to run my own instance?"* + +\.\. is a common question among searx users. Before answering this question, +see what options a searx user has. Public instances are open to everyone who has access to its URL. Usually, these are operated by unknown parties (from the users' point of view). Private diff --git a/docs/user/public_instances.rst b/docs/user/public_instances.rst deleted file mode 100644 index f58ab7a02..000000000 --- a/docs/user/public_instances.rst +++ /dev/null @@ -1,3 +0,0 @@ -:orphan: - -This page page has been moved to `searx.space <https://searx.space/>`__ diff --git a/docs/utils/filtron.sh.rst b/docs/utils/filtron.sh.rst new file mode 100644 index 000000000..1bba1b3e4 --- /dev/null +++ b/docs/utils/filtron.sh.rst @@ -0,0 +1,75 @@ + +.. _filtron.sh: + +==================== +``utils/filtron.sh`` +==================== + +.. sidebar:: further reading + + - :ref:`installation` + - :ref:`searx_filtron` + - :ref:`architecture` + +.. _Go: https://golang.org/ +.. _filtron: https://github.com/asciimoo/filtron +.. _filtron README: https://github.com/asciimoo/filtron/blob/master/README.md + +To simplify installation and maintenance of a filtron instance you can use the +script :origin:`utils/filtron.sh`. In most cases you will install filtron_ +simply by running the command: + +.. code:: bash + + sudo -H ./utils/filtron.sh install all + +The script adds a ``${SERVICE_USER}`` (default:``filtron``) and installs filtron_ +into this user account: + +#. Create a separated user account (``filtron``). +#. Download and install Go_ binary in users $HOME (``~filtron``). +#. Install filtron with the package management of Go_ (``go get -v -u + github.com/asciimoo/filtron``) +#. Setup a proper rule configuration :origin:`[ref] + <utils/templates/etc/filtron/rules.json>` (``/etc/filtron/rules.json``). +#. Setup a systemd service unit :origin:`[ref] + <utils/templates/lib/systemd/system/filtron.service>` + (``/lib/systemd/system/filtron.service``). + +.. _reverse proxy: + +Public Reverse Proxy +==================== + +To install searx in your public HTTP server use: + +.. code:: bash + + sudo -H ./utils/filtron.sh apache install + +.. tabs:: + + .. group-tab:: apache + + .. literalinclude:: ../../utils/templates/etc/apache2/sites-available/searx.conf:filtron + :language: apache + + .. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code-block:: sh + + $ sudo -H a2enmod headers + $ sudo -H a2enmod proxy + $ sudo -H a2enmod proxy_http + + +Overview +======== + +The ``--help`` output of the script is largely self-explanatory +(:ref:`toolboxing common`): + +.. program-output:: ../utils/filtron.sh --help + diff --git a/docs/utils/index.rst b/docs/utils/index.rst new file mode 100644 index 000000000..338914715 --- /dev/null +++ b/docs/utils/index.rst @@ -0,0 +1,53 @@ + +.. _searx_utils: +.. _toolboxing: + +======================= +Tooling box ``utils/*`` +======================= + +In the folder :origin:`utils/` we maintain some tools useful for admins and +developers. + +.. sidebar:: Work needed! + + Our scripts to maintain services do most support only systemd init process + used by debian, ubuntu and many other dists. In general our scripts are only + partially usable on debian systems. We are working on this limitation, if + you have any contribution, please send us your :pull:`PR <../pulls>`, see + :ref:`how to contribute`. + +.. toctree:: + :maxdepth: 2 + :caption: Contents + + searx.sh + filtron.sh + morty.sh + +.. _toolboxing common: + +Common commands +=============== + +Scripts to maintain services often dispose of common commands and environments. + +``shell``: + Opens a shell from the service user ``${SERVICE_USSR}``, very helpful for + troubleshooting. + +``inspect service``: + Shows status and log of the service, most often you have a option to enable + more verbose debug logs. Very helpful for debugging, but be careful not to + enable debugging in a production environment! + +.. _toolboxing setup: + +Tooling box setup +================= + +The main setup is done in the :origin:`.config.sh` (read also :ref:`makefile +setup`). + +.. literalinclude:: ../../.config.sh + :language: bash diff --git a/docs/utils/morty.sh.rst b/docs/utils/morty.sh.rst new file mode 100644 index 000000000..9997ffec5 --- /dev/null +++ b/docs/utils/morty.sh.rst @@ -0,0 +1,39 @@ + +.. _morty: https://github.com/asciimoo/morty +.. _morty's README: https://github.com/asciimoo/morty + +.. _morty.sh: + +================== +``utils/morty.sh`` +================== + +.. sidebar:: further reading + + - :ref:`installation` + - :ref:`architecture` + +To simplify installation and maintenance of a morty_ instance you can use the +script :origin:`utils/morty.sh`. In most cases you will install morty_ simply by +running the command: + +.. code:: bash + + sudo -H ./utils/morty.sh install all + +The script adds a ``${SERVICE_USER}`` (default:``morty``) and installs morty_ +into this user account. + +.. hint:: + + To add morty to your searx instance read chapter :reF:`searx_morty`. + + +Overview +======== + +The ``--help`` output of the script is largely self-explanatory +(:ref:`toolboxing common`): + +.. program-output:: ../utils/morty.sh --help + diff --git a/docs/utils/searx.sh.rst b/docs/utils/searx.sh.rst new file mode 100644 index 000000000..988c08438 --- /dev/null +++ b/docs/utils/searx.sh.rst @@ -0,0 +1,73 @@ + +.. _searx.sh: + +================== +``utils/searx.sh`` +================== + +.. sidebar:: further reading + + - :ref:`installation` + - :ref:`architecture` + - :ref:`filtron.sh` + +To simplify installation and maintenance of a searx instance you can use the +script :origin:`utils/searx.sh`. + +Install +======= + +In most cases you will install searx simply by running the command: + +.. code:: bash + + sudo -H ./utils/searx.sh install all + +The script adds a ``${SERVICE_USER}`` (default:``searx``) and installs searx +into this user account. The installation is described in chapter +:ref:`installation basic`. + +.. _intranet reverse proxy: + +Intranet Reverse Proxy +====================== + +To install searx in your intranet HTTP server use: + +.. code:: bash + + sudo -H ./utils/searx.sh apache install + +.. warning:: + + This setup is **not** suitable **for public instances**, go on with + :ref:`reverse proxy`! + +.. tabs:: + + .. group-tab:: apache + + .. literalinclude:: ../../utils/templates/etc/apache2/sites-available/searx.conf:uwsgi + :language: apache + + .. tabs:: + + .. group-tab:: Ubuntu / debian + + .. code-block:: sh + + $ sudo -H apt install libapache2-mod-uwsgi + + .. group-tab:: Arch Linux + + .. code-block:: sh + + $ sudo pacman -S uwsgi + +Overview +======== + +The ``--help`` output of the script is largely self-explanatory +(:ref:`toolboxing common`): + +.. program-output:: ../utils/searx.sh --help diff --git a/requirements-dev.txt b/requirements-dev.txt index 3e8f617af..a794c3e10 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -14,3 +14,5 @@ selenium==3.141.0 linuxdoc @ git+http://github.com/return42/linuxdoc.git sphinx-jinja sphinx-tabs +sphinxcontrib-programoutput +twine diff --git a/utils/filtron.sh b/utils/filtron.sh new file mode 100755 index 000000000..13bb76751 --- /dev/null +++ b/utils/filtron.sh @@ -0,0 +1,451 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- +# SPDX-License-Identifier: AGPL-3.0-or-later +# shellcheck disable=SC2119,SC2001 + +# shellcheck source=utils/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +source_dot_config + +# ---------------------------------------------------------------------------- +# config +# ---------------------------------------------------------------------------- + +FILTRON_URL_PATH="${FILTRON_URL_PATH:-$(echo "${PUBLIC_URL}" \ +| sed -e 's,^.*://[^/]*\(/.*\),\1,g')}" +[[ "${FILTRON_URL_PATH}" == "${PUBLIC_URL}" ]] && FILTRON_URL_PATH=/ + +FILTRON_ETC="/etc/filtron" + +FILTRON_RULES="$FILTRON_ETC/rules.json" + +FILTRON_API="${FILTRON_API:-127.0.0.1:4005}" +FILTRON_LISTEN="${FILTRON_LISTEN:-127.0.0.1:4004}" +FILTRON_TARGET="${FILTRON_TARGET:-127.0.0.1:8888}" + +SERVICE_NAME="filtron" +SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}" +SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}" +SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}" +SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service" +# shellcheck disable=SC2034 +SERVICE_GROUP="${SERVICE_USER}" + +# shellcheck disable=SC2034 +SERVICE_GROUP="${SERVICE_USER}" + +GO_ENV="${SERVICE_HOME}/.go_env" +GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz" +GO_TAR=$(basename "$GO_PKG_URL") + +# Apache Settings + +APACHE_FILTRON_SITE="searx.conf" + +# shellcheck disable=SC2034 +CONFIG_FILES=( + "${FILTRON_RULES}" + "${SERVICE_SYSTEMD_UNIT}" +) + +# ---------------------------------------------------------------------------- +usage() { +# ---------------------------------------------------------------------------- + + # shellcheck disable=SC1117 + cat <<EOF + +usage:: + + $(basename "$0") shell + $(basename "$0") install [all|user|rules] + $(basename "$0") update [filtron] + $(basename "$0") remove [all] + $(basename "$0") activate [service] + $(basename "$0") deactivate [service] + $(basename "$0") inspect [service] + $(basename "$0") option [debug-on|debug-off] + $(basename "$0") apache [install|remove] + +shell + start interactive shell from user ${SERVICE_USER} +install / remove + :all: complete setup of filtron service + :user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME) + :rules: reinstall filtron rules $FILTRON_RULES +update filtron + Update filtron installation ($SERVICE_HOME) +activate service + activate and start service daemon (systemd unit) +deactivate service + stop and deactivate service daemon (systemd unit) +inspect service + show service status and log +option + set one of the available options +apache : ${PUBLIC_URL} + :install: apache site with a reverse proxy (ProxyPass) + :remove: apache site ${APACHE_FILTRON_SITE} + +If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/"}' file:: + + PUBLIC_URL : ${PUBLIC_URL} + PUBLIC_HOST : ${PUBLIC_HOST} + SERVICE_USER : ${SERVICE_USER} + FILTRON_API : ${FILTRON_API} + FILTRON_LISTEN : ${FILTRON_LISTEN} + FILTRON_TARGET : ${FILTRON_TARGET} + +EOF + [[ -n ${1} ]] && err_msg "$1" +} + +main() { + rst_title "$SERVICE_NAME" part + + required_commands \ + dpkg apt-get install git wget curl \ + || exit + + local _usage="unknown or missing $1 command $2" + + case $1 in + --source-only) ;; + -h|--help) usage; exit 0;; + + shell) + sudo_or_exit + interactive_shell "${SERVICE_USER}" + ;; + inspect) + case $2 in + service) + sudo_or_exit + inspect_service + ;; + *) usage "$_usage"; exit 42;; + esac ;; + install) + sudo_or_exit + case $2 in + all) install_all ;; + user) assert_user ;; + rules) + rst_title "Re-Install filtron rules" + echo + install_template --no-eval "$FILTRON_RULES" root root 644 + systemd_restart_service "${SERVICE_NAME}" + ;; + *) usage "$_usage"; exit 42;; + esac ;; + update) + sudo_or_exit + case $2 in + filtron) update_filtron ;; + *) usage "$_usage"; exit 42;; + esac ;; + remove) + sudo_or_exit + case $2 in + all) remove_all;; + user) drop_service_account "${SERVICE_USER}" ;; + *) usage "$_usage"; exit 42;; + esac ;; + activate) + sudo_or_exit + case $2 in + service) systemd_activate_service "${SERVICE_NAME}" ;; + *) usage "$_usage"; exit 42;; + esac ;; + deactivate) + sudo_or_exit + case $2 in + service) systemd_deactivate_service "${SERVICE_NAME}" ;; + *) usage "$_usage"; exit 42;; + esac ;; + apache) + sudo_or_exit + case $2 in + install) install_apache_site ;; + remove) remove_apache_site ;; + *) usage "$_usage"; exit 42;; + esac ;; + option) + sudo_or_exit + case $2 in + debug-on) echo; enable_debug ;; + debug-off) echo; disable_debug ;; + *) usage "$_usage"; exit 42;; + esac ;; + + *) usage "unknown or missing command $1"; exit 42;; + esac +} + +install_all() { + rst_title "Install $SERVICE_NAME (service)" + assert_user + wait_key + install_go "${GO_PKG_URL}" "${GO_TAR}" "${SERVICE_USER}" + wait_key + install_filtron + wait_key + systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" + wait_key + echo + if ! service_is_available "http://${FILTRON_LISTEN}" ; then + err_msg "Filtron does not listening on: http://${FILTRON_LISTEN}" + fi + if apache_is_installed; then + info_msg "Apache is installed on this host." + if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then + install_apache_site + fi + fi + if ask_yn "Do you want to inspect the installation?" Yn; then + inspect_service + fi + +} + +remove_all() { + rst_title "De-Install $SERVICE_NAME (service)" + + rst_para "\ +It goes without saying that this script can only be used to remove +installations that were installed with this script." + + if ! systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then + return 42 + fi + drop_service_account "${SERVICE_USER}" + rm -r "$FILTRON_ETC" 2>&1 | prefix_stdout + if service_is_available "${PUBLIC_URL}"; then + MSG="** Don't forget to remove your public site! (${PUBLIC_URL}) **" wait_key 10 + fi +} + +assert_user() { + rst_title "user $SERVICE_USER" section + echo + tee_stderr 1 <<EOF | bash | prefix_stdout +sudo -H adduser --shell /bin/bash --system --home $SERVICE_HOME \ + --disabled-password --group --gecos 'Filtron' $SERVICE_USER +sudo -H usermod -a -G shadow $SERVICE_USER +groups $SERVICE_USER +EOF + SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)" + export SERVICE_HOME + echo "export SERVICE_HOME=$SERVICE_HOME" + + cat > "$GO_ENV" <<EOF +export GOPATH=\$HOME/go-apps +export PATH=\$PATH:\$HOME/local/go/bin:\$GOPATH/bin +EOF + echo "Environment $GO_ENV has been setup." + + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" +grep -qFs -- 'source $GO_ENV' ~/.profile || echo 'source $GO_ENV' >> ~/.profile +EOF +} + + +filtron_is_installed() { + [[ -f $SERVICE_HOME/go-apps/bin/filtron ]] +} + +_svcpr=" |${SERVICE_USER}| " + +install_filtron() { + rst_title "Install filtron in user's ~/go-apps" section + echo + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr" +go get -v -u github.com/asciimoo/filtron +EOF + install_template --no-eval "$FILTRON_RULES" root root 644 +} + +update_filtron() { + rst_title "Update filtron" section + echo + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr" +go get -v -u github.com/asciimoo/filtron +EOF +} + +inspect_service() { + + rst_title "service status & log" + + cat <<EOF + +sourced ${DOT_CONFIG#"$REPO_ROOT/"} : + + PUBLIC_URL : ${PUBLIC_URL} + PUBLIC_HOST : ${PUBLIC_HOST} + FILTRON_URL_PATH : ${FILTRON_URL_PATH} + FILTRON_API : ${FILTRON_API} + FILTRON_LISTEN : ${FILTRON_LISTEN} + FILTRON_TARGET : ${FILTRON_TARGET} + +EOF + + apache_is_installed && info_msg "Apache is installed." + + if service_account_is_available "$SERVICE_USER"; then + info_msg "service account $SERVICE_USER available." + else + err_msg "service account $SERVICE_USER not available!" + fi + if go_is_available "$SERVICE_USER"; then + info_msg "~$SERVICE_USER: go is installed" + else + err_msg "~$SERVICE_USER: go is not installed" + fi + if filtron_is_installed; then + info_msg "~$SERVICE_USER: filtron app is installed" + else + err_msg "~$SERVICE_USER: filtron app is not installed!" + fi + + if ! service_is_available "http://${FILTRON_API}"; then + err_msg "API not available at: http://${FILTRON_API}" + fi + + if ! service_is_available "http://${FILTRON_LISTEN}" ; then + err_msg "Filtron does not listening on: http://${FILTRON_LISTEN}" + fi + + if service_is_available "http://${FILTRON_TARGET}" ; then + info_msg "Filtron's target is available at: http://${FILTRON_TARGET}" + fi + + if ! service_is_available "${PUBLIC_URL}"; then + err_msg "Public service at ${PUBLIC_URL} is not available!" + wait_key + fi + + local _debug_on + if ask_yn "Enable filtron debug mode?"; then + enable_debug + _debug_on=1 + fi + + echo + systemctl --no-pager -l status "${SERVICE_NAME}" + echo + + info_msg "public URL --> ${PUBLIC_URL}" + # shellcheck disable=SC2059 + printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log" + read -r -s -n1 -t 2 + echo + while true; do + trap break 2 + journalctl -f -u "${SERVICE_NAME}" + done + + if [[ $_debug_on == 1 ]]; then + disable_debug + fi + return 0 +} + + +enable_debug() { + info_msg "try to enable debug mode ..." + python <<EOF +import sys, json + +debug = { + u'name': u'debug request' + , u'filters': [] + , u'interval': 0 + , u'limit': 0 + , u'actions': [{u'name': u'log'}] +} + +with open('$FILTRON_RULES') as rules: + j = json.load(rules) + +pos = None +for i in range(len(j)): + if j[i].get('name') == 'debug request': + pos = i + break +if pos is not None: + j[pos] = debug +else: + j.append(debug) +with open('$FILTRON_RULES', 'w') as rules: + json.dump(j, rules, indent=2, sort_keys=True) + +EOF + systemctl restart "${SERVICE_NAME}.service" +} + +disable_debug() { + info_msg "try to disable debug mode ..." + python <<EOF +import sys, json +with open('$FILTRON_RULES') as rules: + j = json.load(rules) + +pos = None +for i in range(len(j)): + if j[i].get('name') == 'debug request': + pos = i + break +if pos is not None: + del j[pos] + with open('$FILTRON_RULES', 'w') as rules: + json.dump(j, rules, indent=2, sort_keys=True) +EOF + systemctl restart "${SERVICE_NAME}.service" +} + +install_apache_site() { + + rst_title "Install Apache site $APACHE_FILTRON_SITE" + + rst_para "\ +This installs a reverse proxy (ProxyPass) into apache site (${APACHE_FILTRON_SITE})" + + ! apache_is_installed && err_msg "Apache is not installed." + + if ! ask_yn "Do you really want to continue?"; then + return + fi + + a2enmod headers + a2enmod proxy + a2enmod proxy_http + + echo + apache_install_site --variant=filtron "${APACHE_FILTRON_SITE}" + + info_msg "testing public url .." + if ! service_is_available "${PUBLIC_URL}"; then + err_msg "Public service at ${PUBLIC_URL} is not available!" + fi +} + +remove_apache_site() { + + rst_title "Remove Apache site $APACHE_FILTRON_SITE" + + rst_para "\ +This removes apache site ${APACHE_FILTRON_SITE}." + + ! apache_is_installed && err_msg "Apache is not installed." + + if ! ask_yn "Do you really want to continue?"; then + return + fi + + apache_remove_site "$APACHE_FILTRON_SITE" +} + +# ---------------------------------------------------------------------------- +main "$@" +# ---------------------------------------------------------------------------- diff --git a/utils/lib.sh b/utils/lib.sh new file mode 100755 index 000000000..a045c91a8 --- /dev/null +++ b/utils/lib.sh @@ -0,0 +1,877 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- +# SPDX-License-Identifier: AGPL-3.0-or-later +# shellcheck disable=SC2059,SC1117 + +ADMIN_NAME="${ADMIN_NAME:-$(git config user.name)}" +ADMIN_NAME="${ADMIN_NAME:-$USER}" + +ADMIN_EMAIL="${ADMIN_EMAIL:-$(git config user.email)}" +ADMIN_EMAIL="${ADMIN_EMAIL:-$USER@$(hostname)}" + +if [[ -z "${REPO_ROOT}" ]]; then + REPO_ROOT=$(dirname "${BASH_SOURCE[0]}") + while [ -h "${REPO_ROOT}" ] ; do + REPO_ROOT=$(readlink "${REPO_ROOT}") + done + REPO_ROOT=$(cd "${REPO_ROOT}/.." && pwd -P ) +fi + +if [[ -z ${TEMPLATES} ]]; then + TEMPLATES="${REPO_ROOT}/utils/templates" +fi + +if [[ -z "$CACHE" ]]; then + CACHE="${REPO_ROOT}/cache" +fi + +if [[ -z ${DIFF_CMD} ]]; then + DIFF_CMD="diff -u" + if command -v colordiff >/dev/null; then + DIFF_CMD="colordiff -u" + fi +fi + +DOT_CONFIG="${DOT_CONFIG:-${REPO_ROOT}/.config.sh}" + +source_dot_config() { + if [[ ! -e "${DOT_CONFIG}" ]]; then + err_msg "configuration does not extsts at: ${DOT_CONFIG}" + return 42 + fi + # shellcheck disable=SC1090 + source "${DOT_CONFIG}" +} + +sudo_or_exit() { + # usage: sudo_or_exit + + if [ ! "$(id -u)" -eq 0 ]; then + err_msg "this command requires root (sudo) privilege!" >&2 + exit 42 + fi +} + +required_commands() { + + # usage: requires_commands [cmd1 ...] + + local exit_val=0 + while [ -n "$1" ]; do + + if ! command -v "$1" &>/dev/null; then + err_msg "missing command $1" + exit_val=42 + fi + shift + done + return $exit_val +} + +# colors +# ------ + +# shellcheck disable=SC2034 +set_terminal_colors() { + _colors=8 + _creset='\e[0m' # reset all attributes + + _Black='\e[0;30m' + _White='\e[1;37m' + _Red='\e[0;31m' + _Green='\e[0;32m' + _Yellow='\e[0;33m' + _Blue='\e[0;34m' + _Violet='\e[0;35m' + _Cyan='\e[0;36m' + + _BBlack='\e[1;30m' + _BWhite='\e[1;37m' + _BRed='\e[1;31m' + _BGreen='\e[1;32m' + _BYellow='\e[1;33m' + _BBlue='\e[1;34m' + _BPurple='\e[1;35m' + _BCyan='\e[1;36m' +} + +if [ ! -p /dev/stdout ]; then + set_terminal_colors +fi + +# reST +# ---- + +if command -v fmt >/dev/null; then + export FMT="fmt -u" +else + export FMT="cat" +fi + +rst_title() { + # usage: rst_title <header-text> [part|chapter|section] + + case ${2-chapter} in + part) printf "\n${_BGreen}${1//?/=}\n${_BCyan}${1}${_BGreen}\n${1//?/=}${_creset}\n";; + chapter) printf "\n${_BCyan}${1}\n${_BGreen}${1//?/=}${_creset}\n";; + section) printf "\n${_BCyan}${1}\n${_BGreen}${1//?/-}${_creset}\n";; + *) + err_msg "invalid argument '${2}' in line $(caller)" + return 42 + ;; + esac +} + +rst_para() { + # usage: RST_INDENT=1 rst_para "lorem ipsum ..." + local prefix='' + if [[ -n $RST_INDENT ]] && [[ $RST_INDENT -gt 0 ]]; then + prefix="$(for i in $(seq 1 "$RST_INDENT"); do printf " "; done)" + echo -en "\n$*\n" | $FMT | prefix_stdout "$prefix" + else + echo -en "\n$*\n" | $FMT + fi +} + +err_msg() { echo -e "${_BRed}ERROR:${_creset} $*" >&2; } +warn_msg() { echo -e "${_BBlue}WARN:${_creset} $*" >&2; } +info_msg() { echo -e "${_BYellow}INFO:${_creset} $*" >&2; } + +clean_stdin() { + if [[ $(uname -s) != 'Darwin' ]]; then + while read -r -n1 -t 0.1; do : ; done + fi +} + +wait_key(){ + # usage: waitKEY [<timeout in sec>] + + clean_stdin + local _t=$1 + local msg="${MSG}" + [[ -z "$msg" ]] && msg="${_Green}** press any [${_BCyan}KEY${_Green}] to continue **${_creset}" + + [[ -n $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT + [[ -n $_t ]] && _t="-t $_t" + printf "$msg" + # shellcheck disable=SC2086 + read -r -s -n1 $_t + echo + clean_stdin +} + +ask_yn() { + # usage: ask_yn <prompt-text> [Ny|Yn] [<timeout in sec>] + + local EXIT_YES=0 # exit status 0 --> successful + local EXIT_NO=1 # exit status 1 --> error code + + local _t=$3 + [[ -n $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT + [[ -n $_t ]] && _t="-t $_t" + case "${FORCE_SELECTION:-${2}}" in + Y) return ${EXIT_YES} ;; + N) return ${EXIT_NO} ;; + Yn) + local exit_val=${EXIT_YES} + local choice="[${_BGreen}YES${_creset}/no]" + local default="Yes" + ;; + *) + local exit_val=${EXIT_NO} + local choice="[${_BGreen}NO${_creset}/yes]" + local default="No" + ;; + esac + echo + while true; do + clean_stdin + printf "$1 ${choice} " + # shellcheck disable=SC2086 + read -r -n1 $_t + if [[ -z $REPLY ]]; then + printf "$default\n"; break + elif [[ $REPLY =~ ^[Yy]$ ]]; then + exit_val=${EXIT_YES} + printf "\n" + break + elif [[ $REPLY =~ ^[Nn]$ ]]; then + exit_val=${EXIT_NO} + printf "\n" + break + fi + _t="" + err_msg "invalid choice" + done + clean_stdin + return $exit_val +} + +tee_stderr () { + + # usage:: + # tee_stderr 1 <<EOF | python -i + # print("hello") + # EOF + # ... + # >>> print("hello") + # hello + + local _t="0"; + if [[ -n $1 ]] ; then _t="$1"; fi + + (while read -r line; do + # shellcheck disable=SC2086 + sleep $_t + echo -e "$line" >&2 + echo "$line" + done) +} + +prefix_stdout () { + # usage: <cmd> | prefix_stdout [prefix] + + local prefix="${_BYellow}-->|${_creset}" + + if [[ -n $1 ]] ; then prefix="${_BYellow}$1${_creset}"; fi + + # shellcheck disable=SC2162 + (while IFS= read line; do + echo -e "${prefix}$line" + done) +} + +append_line() { + + # usage: append_line <line> <file> + # + # Append line if not exists, create file if not exists. E.g:: + # + # append_line 'source ~/.foo' ~/bashrc + + local LINE=$1 + local FILE=$2 + grep -qFs -- "$LINE" "$FILE" || echo "$LINE" >> "$FILE" +} + +cache_download() { + + # usage: cache_download <url> <local-filename> + + local exit_value=0 + + if [[ -n ${SUDO_USER} ]]; then + sudo -u "${SUDO_USER}" mkdir -p "${CACHE}" + else + mkdir -p "${CACHE}" + fi + + if [[ -f "${CACHE}/$2" ]] ; then + info_msg "already cached: $1" + info_msg " --> ${CACHE}/$2" + fi + + if [[ ! -f "${CACHE}/$2" ]]; then + info_msg "caching: $1" + info_msg " --> ${CACHE}/$2" + if [[ -n ${SUDO_USER} ]]; then + sudo -u "${SUDO_USER}" wget --progress=bar -O "${CACHE}/$2" "$1" ; exit_value=$? + else + wget --progress=bar -O "${CACHE}/$2" "$1" ; exit_value=$? + fi + if [[ ! $exit_value = 0 ]]; then + err_msg "failed to download: $1" + fi + fi +} + +choose_one() { + + # usage: + # + # DEFAULT_SELECT= 2 \ + # choose_one <name> "your selection?" "Coffee" "Coffee with milk" + + local default=${DEFAULT_SELECT-1} + local REPLY + local env_name=$1 && shift + local choice=$1; + local max="${#@}" + local _t + [[ -n $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT + [[ -n $_t ]] && _t="-t $_t" + + list=("$@") + echo -e "${_BGreen}Menu::${_creset}" + for ((i=1; i<= $((max -1)); i++)); do + if [[ "$i" == "$default" ]]; then + echo -e " ${_BGreen}$i.${_creset}) ${list[$i]} [default]" + else + echo -e " $i.) ${list[$i]}" + fi + done + while true; do + clean_stdin + printf "$1 [${_BGreen}$default${_creset}] " + + if (( 10 > max )); then + # shellcheck disable=SC2086 + read -r -n1 $_t + else + # shellcheck disable=SC2086,SC2229 + read -r $_t + fi + # selection fits + [[ $REPLY =~ ^-?[0-9]+$ ]] && (( REPLY > 0 )) && (( REPLY < max )) && break + + # take default + [[ -z $REPLY ]] && REPLY=$default && break + + _t="" + err_msg "invalid choice" + done + eval "$env_name"='${list[${REPLY}]}' + echo + clean_stdin +} + +install_template() { + + # usage: + # + # install_template [--no-eval] [--variant=<name>] \ + # {file} [{owner} [{group} [{chmod}]]] + # + # E.g. the origin of variant 'raw' of /etc/updatedb.conf is:: + # + # ${TEMPLATES}/etc/updatedb.conf:raw + # + # To install variant 'raw' of /etc/updatedb.conf without evaluated + # replacements you can use:: + # + # install_template --variant=raw --no-eval \ + # /etc/updatedb.conf root root 644 + + local _reply="" + local do_eval=1 + local variant="" + local pos_args=("$0") + + for i in "$@"; do + case $i in + --no-eval) do_eval=0; shift ;; + --variant=*) variant=":${i#*=}"; shift ;; + *) pos_args+=("$i") ;; + esac + done + + local dst="${pos_args[1]}" + local template_origin="${TEMPLATES}${dst}${variant}" + local template_file="${TEMPLATES}${dst}" + + local owner="${pos_args[2]-$(id -un)}" + local group="${pos_args[3]-$(id -gn)}" + local chmod="${pos_args[4]-644}" + + info_msg "install (eval=$do_eval): ${dst}" + [[ -n $variant ]] && info_msg "variant: ${variant}" + + if [[ ! -f "${template_origin}" ]] ; then + err_msg "${template_origin} does not exists" + err_msg "... can't install $dst" + wait_key 30 + return 42 + fi + + if [[ "$do_eval" == "1" ]]; then + template_file="${CACHE}${dst}${variant}" + info_msg "BUILD template ${template_file}" + if [[ -n ${SUDO_USER} ]]; then + sudo -u "${SUDO_USER}" mkdir -p "$(dirname "${template_file}")" + else + mkdir -p "$(dirname "${template_file}")" + fi + # shellcheck disable=SC2086 + eval "echo \"$(cat ${template_origin})\"" > "${template_file}" + if [[ -n ${SUDO_USER} ]]; then + chown "${SUDO_USER}:${SUDO_USER}" "${template_file}" + fi + else + template_file=$template_origin + fi + + mkdir -p "$(dirname "${dst}")" + + if [[ ! -f "${dst}" ]]; then + info_msg "install: ${template_file}" + sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \ + "${template_file}" "${dst}" | prefix_stdout + return $? + fi + + if [[ -f "${dst}" ]] && cmp --silent "${template_file}" "${dst}" ; then + info_msg "file ${dst} allready installed" + return 0 + fi + + info_msg "diffrent file ${dst} allready exists on this host" + + while true; do + choose_one _reply "choose next step with file $dst" \ + "replace file" \ + "leave file unchanged" \ + "interactiv shell" \ + "diff files" + + case $_reply in + "replace file") + info_msg "install: ${template_file}" + sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \ + "${template_file}" "${dst}" | prefix_stdout + break + ;; + "leave file unchanged") + break + ;; + "interactiv shell") + echo "// edit ${dst} to your needs" + echo -e "// exit with [${_BCyan}CTRL-D${_creset}]" + sudo -H -u "${owner}" -i + $DIFF_CMD "${dst}" "${template_file}" + echo + echo "${_BBlack}did you edit file ...${_creset}" + printf " ${template_file}" + if ask_yn "... to your needs?"; then + break + fi + ;; + "diff files") + $DIFF_CMD "${dst}" "${template_file}" | prefix_stdout + esac + done +} + + +service_is_available() { + + # usage: service_is_available <URL> + + local URL="$1" + if [[ -z $URL ]]; then + err_msg "service_is_available: missing arguments" + return 42 + fi + + http_code=$(curl -H 'Cache-Control: no-cache' \ + --silent -o /dev/null --head --write-out '%{http_code}' --insecure \ + "${URL}") + exit_val=$? + if [[ $exit_val = 0 ]]; then + info_msg "got $http_code from ${URL}" + fi + case "$http_code" in + 404|410|423) exit_val=$http_code;; + esac + return "$exit_val" +} + +# golang +# ------ + +go_is_available() { + + # usage: go_is_available $SERVICE_USER && echo "go is installed!" + + sudo -i -u "${1}" which go &>/dev/null +} + +install_go() { + + # usage: install_go "${GO_PKG_URL}" "${GO_TAR}" "${SERVICE_USER}" + + local _svcpr=" |${3}| " + + rst_title "Install Go in user's HOME" section + + rst_para "download and install go binary .." + cache_download "${1}" "${2}" + + tee_stderr 0.1 <<EOF | sudo -i -u "${3}" | prefix_stdout "$_svcpr" +echo \$PATH +echo \$GOPATH +mkdir -p \$HOME/local +rm -rf \$HOME/local/go +tar -C \$HOME/local -xzf ${CACHE}/${2} +EOF + sudo -i -u "${3}" <<EOF | prefix_stdout +! which go >/dev/null && echo "ERROR - Go Installation not found in PATH!?!" +which go >/dev/null && go version && echo "congratulations -- Go installation OK :)" +EOF +} + +# system accounts +# --------------- + +service_account_is_available() { + + # usage: service_account_is_available "$SERVICE_USER" && echo "OK" + + sudo -i -u "$1" echo \$HOME &>/dev/null +} + +drop_service_account() { + + # usage: drop_service_account "${SERVICE_USER}" + + rst_title "Drop ${1} HOME" section + if ask_yn "Do you really want to drop ${1} home folder?"; then + userdel -r -f "${1}" 2>&1 | prefix_stdout + else + rst_para "Leave HOME folder $(du -sh "${1}") unchanged." + fi +} + +interactive_shell(){ + + # usage: interactive_shell "${SERVICE_USER}" + + echo -e "// exit with [${_BCyan}CTRL-D${_creset}]" + sudo -H -u "${1}" -i +} + + +# systemd +# ------- + +SYSTEMD_UNITS="${SYSTEMD_UNITS:-/lib/systemd/system}" + +systemd_install_service() { + + # usage: systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" + + rst_title "Install System-D Unit ${1}" section + echo + install_template "${2}" root root 644 + wait_key + systemd_activate_service "${1}" +} + +systemd_remove_service() { + + # usage: systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" + + if ! ask_yn "Do you really want to deinstall systemd unit ${1}?"; then + return 42 + fi + systemd_deactivate_service "${1}" + rm "${2}" 2>&1 | prefix_stdout +} + +systemd_activate_service() { + + # usage: systemd_activate_service "${SERVICE_NAME}" + + rst_title "Activate ${1} (service)" section + echo + tee_stderr <<EOF | bash 2>&1 +systemctl enable ${1}.service +systemctl restart ${1}.service +EOF + tee_stderr <<EOF | bash 2>&1 +systemctl status --no-pager ${1}.service +EOF +} + +systemd_deactivate_service() { + + # usage: systemd_deactivate_service "${SERVICE_NAME}" + + rst_title "De-Activate ${1} (service)" section + echo + tee_stderr <<EOF | bash 2>&1 | prefix_stdout +systemctl stop ${1}.service +systemctl disable ${1}.service +EOF +} + +systemd_restart_service() { + + # usage: systemd_restart_service "${SERVICE_NAME}" + + rst_title "Restart ${1} (service)" section + echo + tee_stderr <<EOF | bash 2>&1 +systemctl restart ${1}.service +EOF + tee_stderr <<EOF | bash 2>&1 +systemctl status --no-pager ${1}.service +EOF +} + + +# Apache +# ------ + +# FIXME: Arch Linux & RHEL should be added + +if [[ -z "${APACHE_SITES_AVAILABE}" ]]; then + APACHE_SITES_AVAILABE="/etc/apache2/sites-available" +fi + +apache_is_installed() { + (command -v apachectl \ + && command -v a2ensite \ + && command -v a2dissite ) &>/dev/null +} + +apache_reload() { + + info_msg "reload apache .." + echo + sudo -H apachectl configtest + sudo -H service apache2 force-reload +} + +apache_install_site() { + + # usage: apache_install_site [<template option> ...] <mysite.conf> + # + # <template option>: see install_template + + local template_opts=() + local pos_args=("$0") + + for i in "$@"; do + case $i in + -*) template_opts+=("$i");; + *) pos_args+=("$i");; + esac + done + + install_template "${template_opts[@]}" \ + "${APACHE_SITES_AVAILABE}/${pos_args[1]}" \ + root root 644 + + apache_enable_site "${pos_args[1]}" + info_msg "installed apache site: ${pos_args[1]}" +} + +apache_remove_site() { + + # usage: apache_remove_site <mysite.conf> + + info_msg "remove apache site: $1" + apache_dissable_site "$1" + rm -f "${APACHE_SITES_AVAILABE}/$1" +} + +apache_enable_site() { + + # usage: apache_enable_site <mysite.conf> + + info_msg "enable apache site: $1" + sudo -H a2ensite -q "$1" + apache_reload +} + +apache_dissable_site() { + + # usage: apache_disable_site <mysite.conf> + + info_msg "disable apache site: $1" + sudo -H a2dissite -q "$1" + apache_reload +} + +# uWSGI +# ----- + +uWSGI_SETUP="${uWSGI_SETUP:=/etc/uwsgi}" + +uWSGI_restart() { + + # usage: uWSGI_restart() + + info_msg "restart uWSGI service" + systemctl restart uwsgi +} + +uWSGI_app_available() { + # usage: uWSGI_app_available <myapp.ini> + local CONF="$1" + if [[ -z $CONF ]]; then + err_msg "uWSGI_app_available: missing arguments" + return 42 + fi + [[ -f "${uWSGI_SETUP}/apps-available/${CONF}" ]] +} + +uWSGI_install_app() { + + # usage: uWSGI_install_app [<template option> ...] <myapp.ini> + # + # <template option>: see install_template + + local pos_args=("$0") + + for i in "$@"; do + case $i in + -*) template_opts+=("$i");; + *) pos_args+=("$i");; + esac + done + + install_template "${template_opts[@]}" \ + "${uWSGI_SETUP}/apps-available/${pos_args[1]}" \ + root root 644 + + uWSGI_enable_app "${pos_args[1]}" + uWSGI_restart + info_msg "installed uWSGI app: ${pos_args[1]}" +} + +uWSGI_remove_app() { + + # usage: uWSGI_remove_app <myapp.ini> + + local CONF="$1" + info_msg "remove uWSGI app: ${CONF}" + uWSGI_disable_app "${CONF}" + uWSGI_restart + rm -f "${uWSGI_SETUP}/apps-available/${CONF}" +} + +uWSGI_app_enabled() { + # usage: uWSGI_app_enabled <myapp.ini> + local CONF="$1" + if [[ -z $CONF ]]; then + err_msg "uWSGI_app_enabled: missing arguments" + return 42 + fi + [[ -f "${uWSGI_SETUP}/apps-enabled/${CONF}" ]] +} + +# shellcheck disable=SC2164 +uWSGI_enable_app() { + + # usage: uWSGI_enable_app <myapp.ini> + + local CONF="$1" + if [[ -z $CONF ]]; then + err_msg "uWSGI_enable_app: missing arguments" + return 42 + fi + pushd "${uWSGI_SETUP}/apps-enabled" >/dev/null + rm -f "$CONF" + # shellcheck disable=SC2226 + ln -s "../apps-available/${CONF}" + info_msg "enabled uWSGI app: ${CONF} (restart uWSGI required)" + popd >/dev/null +} + +uWSGI_disable_app() { + + # usage: uWSGI_disable_app <myapp.ini> + + local CONF="$1" + if [[ -z $CONF ]]; then + err_msg "uWSGI_enable_app: missing arguments" + return 42 + fi + rm -f "${uWSGI_SETUP}/apps-enabled/${CONF}" + # FIXME: restart uwsgi service won't stop wsgi forked processes of user searx. + # I had to kill them manually here ... + pkill -f "${uWSGI_SETUP}/apps-enabled/${CONF}" -9 + info_msg "disabled uWSGI app: ${CONF} (restart uWSGI required)" +} + +# distro's package manager +# ------------------------ +# +# FIXME: Arch Linux & RHEL should be added +# + +pkg_install() { + + # usage: TITEL='install foobar' pkg_install foopkg barpkg + + rst_title "${TITLE:-installation of packages}" section + echo -en "\npackage(s)::\n\n $*\n" | $FMT + + if ! ask_yn "Should packages be installed?" Yn 30; then + return 42 + fi + # shellcheck disable=SC2068 + apt-get install -m -y $@ +} + +pkg_remove() { + + # usage: TITEL='remove foobar' pkg_remove foopkg barpkg + + rst_title "${TITLE:-remove packages}" section + echo -en "\npackage(s)::\n\n $*\n" | $FMT + + if ! ask_yn "Should packages be removed (purge)?" Yn 30; then + return 42 + fi + apt-get purge --autoremove --ignore-missing -y "$@" +} + +pkg_is_installed() { + + # usage: pkg_is_install foopkg || pkg_install foopkg + + dpkg -l "$1" &> /dev/null + return $? +} + +# git tooling +# ----------- + +# shellcheck disable=SC2164 +git_clone() { + + # usage: + # + # git_clone <url> <name> [<branch> [<user>]] + # git_clone <url> <path> [<branch> [<user>]] + # + # First form uses $CACHE/<name> as destination folder, second form clones + # into <path>. If repository is allready cloned, pull from <branch> and + # update working tree (if needed, the caller has to stash local changes). + # + # git clone https://github.com/asciimoo/searx searx-src origin/master searxlogin + # + + local url="$1" + local dest="$2" + local branch="$3" + local user="$4" + local bash_cmd="bash" + local remote="origin" + + if [[ ! "${dest:0:1}" = "/" ]]; then + dest="$CACHE/$dest" + fi + + [[ -z $branch ]] && branch=master + [[ -z $user ]] && [[ -n "${SUDO_USER}" ]] && user="${SUDO_USER}" + [[ -n $user ]] && bash_cmd="sudo -H -u $user -i" + + if [[ -d "${dest}" ]] ; then + info_msg "already cloned: $dest" + tee_stderr 0.1 <<EOF | $bash_cmd 2>&1 | prefix_stdout " |$user| " +cd "${dest}" +git checkout -m -B "$branch" --track "$remote/$branch" +git pull --all +EOF + else + info_msg "clone into: $dest" + tee_stderr 0.1 <<EOF | $bash_cmd 2>&1 | prefix_stdout " |$user| " +mkdir -p "$(dirname "$dest")" +cd "$(dirname "$dest")" +git clone --branch "$branch" --origin "$remote" "$url" "$(basename "$dest")" +EOF + fi +} diff --git a/utils/lxc.sh b/utils/lxc.sh new file mode 100755 index 000000000..3d627e8b7 --- /dev/null +++ b/utils/lxc.sh @@ -0,0 +1,328 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- +# SPDX-License-Identifier: AGPL-3.0-or-later + +# shellcheck source=utils/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +source_dot_config + +# ---------------------------------------------------------------------------- +# config +# ---------------------------------------------------------------------------- +# +# read also: +# - https://lxd.readthedocs.io/en/latest/ + +# name of https://images.linuxcontainers.org +LINUXCONTAINERS_ORG_NAME="${LINUXCONTAINERS_ORG_NAME:-images}" +HOST_PREFIX="${HOST_PREFIX:-searx}" + +TEST_IMAGES=( + "$LINUXCONTAINERS_ORG_NAME:ubuntu/18.04" "ubu1804" + "$LINUXCONTAINERS_ORG_NAME:ubuntu/19.04" "ubu1904" + + # TODO: installation of searx & filtron not yet implemented .. + # + #"$LINUXCONTAINERS_ORG_NAME:archlinux" "archlinux" + #"$LINUXCONTAINERS_ORG_NAME:fedora/31" "fedora31" +) + +ubu1804_boilerplate=" +export DEBIAN_FRONTEND=noninteractive +apt-get install -y git curl wget +" +# shellcheck disable=SC2034 +ubu1904_boilerplate="$ubu1804_boilerplate" + +REMOTE_IMAGES=() +LOCAL_IMAGES=() + +for ((i=0; i<${#TEST_IMAGES[@]}; i+=2)); do + REMOTE_IMAGES=("${REMOTE_IMAGES[@]}" "${TEST_IMAGES[i]}") + LOCAL_IMAGES=("${LOCAL_IMAGES[@]}" "${HOST_PREFIX}-${TEST_IMAGES[i+1]}") +done + +HOST_USER="${SUDO_USER:-$USER}" +HOST_USER_ID=$(id -u "${HOST_USER}") +HOST_GROUP_ID=$(id -g "${HOST_USER}") + +# ---------------------------------------------------------------------------- +usage() { +# ---------------------------------------------------------------------------- + + cat <<EOF + +usage:: + + $(basename "$0") build [containers] + $(basename "$0") remove [containers|subordinate] + $(basename "$0") [start|stop] [containers] + $(basename "$0") inspect [info|config] + $(basename "$0") cmd ... + +build / remove + :containers: build and remove all LXC containers +add / remove + :subordinate: lxd permission to map ${HOST_USER}'s user/group id through +start/stop + :containers: start/stop of all containers +inspect + :info: show info of all containers + :config: show config of all containers +cmd ... + run commandline ... in all containers + +all LXC containers: + ${LOCAL_IMAGES[@]} + +EOF + [ -n "${1+x}" ] && err_msg "$1" +} + +lxd_info() { + + cat <<EOF + +LXD is needed, to install run:: + + snap install lxd + lxd init --auto + +EOF +} + +main() { + + local exit_val + + if ! required_commands lxc; then + lxd_info + exit 42 + fi + + local _usage="unknown or missing $1 command $2" + + case $1 in + --source-only) ;; + -h|--help) usage; exit 0;; + + build) + sudo_or_exit + case $2 in + containers) build_instances ;; + *) usage "$_usage"; exit 42;; + esac ;; + remove) + sudo_or_exit + case $2 in + containers) remove_instances ;; + subordinate) echo; del_subordinate_ids ;; + *) usage "$_usage"; exit 42;; + esac ;; + add) + sudo_or_exit + case $2 in + subordinate) echo; add_subordinate_ids ;; + *) usage "$_usage"; exit 42;; + esac ;; + start|stop) + sudo_or_exit + case $2 in + containers) lxc_cmd "$1" ;; + *) usage "$_usage"; exit 42;; + esac ;; + inspect) + sudo_or_exit + case $2 in + config) lxc_cmd config show;; + info) lxc_cmd info;; + *) usage "$_usage"; exit 42;; + esac ;; + cmd) + sudo_or_exit + shift + for i in "${LOCAL_IMAGES[@]}"; do + info_msg "call ${_BBlue}${i}${_creset} -- ${_BGreen}${*}${_creset}" + wait_key 3 + lxc exec "${i}" -- "$@" + exit_val=$? + if [ $exit_val -ne 0 ]; then + err_msg "$exit_val ${_BBlue}${i}${_creset} -- ${_BGreen}${*}${_creset}" + fi + done + ;; + *) + usage "unknown or missing command $1"; exit 42;; + esac +} + +build_instances() { + rst_title "Build LXC instances" + + rst_title "copy images" section + echo + lxc_copy_images_localy + lxc image list local: && wait_key + echo + rst_title "build containers" section + echo + lxc_init_containers + lxc_config_containers + lxc_boilerplate_containers + echo + lxc list "$HOST_PREFIX" +} + +remove_instances() { + rst_title "Remove LXC instances" + echo -en "\\nLXC containers(s)::\\n\\n ${LOCAL_IMAGES[*]}\\n" | $FMT + if ask_yn "Do you really want to delete all images"; then + lxc_delete_containers + fi + echo + lxc list "$HOST_PREFIX" + # lxc image list local: && wait_key +} + +# images +# ------ + +lxc_copy_images_localy() { + for ((i=0; i<${#TEST_IMAGES[@]}; i+=2)); do + if lxc image info "local:${TEST_IMAGES[i+1]}" &>/dev/null; then + info_msg "image ${TEST_IMAGES[i]} already copied --> ${TEST_IMAGES[i+1]}" + else + info_msg "copy image locally ${TEST_IMAGES[i]} --> ${TEST_IMAGES[i+1]}" + lxc image copy "${TEST_IMAGES[i]}" local: \ + --alias "${TEST_IMAGES[i+1]}" | prefix_stdout + fi + done +} + +lxc_delete_images_localy() { + echo + for i in "${LOCAL_IMAGES[@]}"; do + info_msg "delete image 'local:$i'" + lxc image delete "local:$i" + done + #lxc image list local: +} + +# container +# --------- + +lxc_cmd() { + for i in "${LOCAL_IMAGES[@]}"; do + info_msg "lxc $* $i" + lxc "$@" "$i" + done +} + +lxc_init_containers() { + local shortname + for ((i=0; i<${#TEST_IMAGES[@]}; i+=2)); do + shortname="${TEST_IMAGES[i+1]}" + if lxc info "${HOST_PREFIX}-${shortname}" &>/dev/null; then + info_msg "conatiner '$i' already exists" + else + info_msg "create conatiner instance: $i" + lxc init "local:${shortname}" "${HOST_PREFIX}-${shortname}" + fi + done +} + +lxc_config_containers() { + for i in "${LOCAL_IMAGES[@]}"; do + info_msg "configure container: ${_BBlue}${i}${_creset}" + + info_msg "map uid/gid from host to container" + # https://lxd.readthedocs.io/en/latest/userns-idmap/#custom-idmaps + echo -e -n "uid $HOST_USER_ID 1000\\ngid $HOST_GROUP_ID 1000"\ + | lxc config set "$i" raw.idmap - + + info_msg "share ${REPO_ROOT} (repo_share) from HOST into container" + # https://lxd.readthedocs.io/en/latest/instances/#type-disk + lxc config device add "$i" repo_share disk \ + source="${REPO_ROOT}" \ + path="/share/$(basename "${REPO_ROOT}")" &>/dev/null + # lxc config show "$i" && wait_key + done +} + +lxc_boilerplate_containers() { + local shortname + local boilerplate_script + for ((i=0; i<${#TEST_IMAGES[@]}; i+=2)); do + shortname="${TEST_IMAGES[i+1]}" + info_msg "install boilerplate: ${_BBlue}${HOST_PREFIX}-${shortname}${_creset}" + lxc start -q "${HOST_PREFIX}-${shortname}" &>/dev/null + boilerplate_script="${shortname}_boilerplate" + boilerplate_script="${!boilerplate_script}" + if [[ -n "${boilerplate_script}" ]]; then + echo "$boilerplate_script" \ + | lxc exec "${HOST_PREFIX}-${shortname}" -- bash \ + | prefix_stdout " ${HOST_PREFIX}-${shortname} | " + else + warn_msg "no boilerplate for instance '$i'" + fi + done +} + +lxc_delete_containers() { + for i in "${LOCAL_IMAGES[@]}"; do + if lxc info "$i" &>/dev/null; then + info_msg "stop & delete instance '$i'" + lxc stop "$i" &>/dev/null + lxc delete "$i" | prefix_stdout + else + warn_msg "instance '$i' does not exist / can't delete :o" + fi + done +} + +# subordinates +# ------------ +# +# see man: subgid(5), subuid(5), https://lxd.readthedocs.io/en/latest/userns-idmap +# +# E.g. in the HOST you have uid=1001(user) and/or gid=1001(user) :: +# +# root:1001:1 +# +# in the CONTAINER:: +# +# config: +# raw.idmap: | +# uid 1001 1000 +# gid 1001 1000 + +add_subordinate_ids() { + if grep "root:${HOST_USER_ID}:1" /etc/subuid -qs; then + info_msg "lxd already has permission to map ${HOST_USER_ID}'s user/group id through" + else + info_msg "add lxd permission to map ${HOST_USER_ID}'s user/group id through" + usermod --add-subuids "${HOST_USER_ID}-${HOST_USER_ID}" \ + --add-subgids "${HOST_GROUP_ID}-${HOST_GROUP_ID}" root + fi +} + +del_subordinate_ids() { + local out + local exit_value + if grep "root:${HOST_USER_ID}:1" /etc/subuid -qs; then + # TODO: root user is always in use by process 1, how can we remove subordinates? + info_msg "remove lxd permission to map ${HOST_USER_ID}'s user/group id through" + out=$(usermod --del-subuids "${HOST_USER_ID}-${HOST_USER_ID}" --del-subgids "${HOST_GROUP_ID}-${HOST_GROUP_ID}" root 2>&1) + exit_val=$? + if [ $exit_val -ne 0 ]; then + err_msg "$out" + fi + else + info_msg "lxd does not have permission to map ${HOST_USER_ID}'s user/group id through" + fi +} + + +# ---------------------------------------------------------------------------- +main "$@" +# ---------------------------------------------------------------------------- diff --git a/utils/makefile.python b/utils/makefile.python index 4aa9d6b49..789cc4c90 100644 --- a/utils/makefile.python +++ b/utils/makefile.python @@ -192,7 +192,7 @@ quiet_cmd_pytest = TEST $@ # .. _installing: https://packaging.python.org/tutorials/installing-packages/ # quiet_cmd_pybuild = BUILD $@ - cmd_pybuild = $(PY_ENV_BIN)/$(PYTHON) setup.py \ + cmd_pybuild = $(PY_ENV_BIN)/python setup.py \ sdist -d $(PYDIST) \ bdist_wheel --bdist-dir $(PYBUILD) -d $(PYDIST) @@ -284,7 +284,7 @@ pyenv-python: pyenv-install # https://github.com/pypa/twine PHONY += upload-pypi -upload-pypi: pyclean pybuild +upload-pypi: pyclean pyenvinstall pybuild @$(PY_ENV_BIN)/twine upload $(PYDIST)/* .PHONY: $(PHONY) diff --git a/utils/morty.sh b/utils/morty.sh new file mode 100755 index 000000000..bd4ad5901 --- /dev/null +++ b/utils/morty.sh @@ -0,0 +1,406 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- +# SPDX-License-Identifier: AGPL-3.0-or-later + +# shellcheck source=utils/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +source_dot_config + +# ---------------------------------------------------------------------------- +# config +# ---------------------------------------------------------------------------- + +PUBLIC_URL_PATH_MORTY="/morty" +PUBLIC_URL_MORTY="$(dirname "${PUBLIC_URL}")${PUBLIC_URL_PATH_MORTY}" + +MORTY_LISTEN="${MORTY_LISTEN:-127.0.0.1:3000}" +# shellcheck disable=SC2034 +MORTY_TIMEOUT=5 + +SERVICE_NAME="morty" +SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}" +SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}" +SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}" +SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service" +# shellcheck disable=SC2034 +SERVICE_GROUP="${SERVICE_USER}" +# shellcheck disable=SC2034 +SERVICE_ENV_DEBUG=false + +GO_ENV="${SERVICE_HOME}/.go_env" +GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz" +GO_TAR=$(basename "$GO_PKG_URL") + +# shellcheck disable=SC2034 +CONFIG_FILES=() + +# Apache Settings + +APACHE_MORTY_SITE="morty.conf" + +# ---------------------------------------------------------------------------- +usage() { +# ---------------------------------------------------------------------------- + + # shellcheck disable=SC1117 + cat <<EOF + +usage:: + + $(basename "$0") shell + $(basename "$0") install [all|user] + $(basename "$0") update [morty] + $(basename "$0") remove [all] + $(basename "$0") activate [service] + $(basename "$0") deactivate [service] + $(basename "$0") inspect [service] + $(basename "$0") option [debug-on|debug-off] + $(basename "$0") apache [install|remove] + $(basename "$0") info [searx] + +shell + start interactive shell from user ${SERVICE_USER} +install / remove + all: complete setup of morty service + user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME) +update morty + Update morty installation ($SERVICE_HOME) +activate service + activate and start service daemon (systemd unit) +deactivate service + stop and deactivate service daemon (systemd unit) +inspect service + show service status and log +option + set one of the available options +apache : ${PUBLIC_URL_MORTY} + :install: apache site with a reverse proxy (ProxyPass) + :remove: apache site ${APACHE_MORTY_SITE} + +If needed, set the environment variable MORTY_LISTEN in the +${DOT_CONFIG#"$REPO_ROOT/"} file:: + + MORTY_LISTEN : ${MORTY_LISTEN} + SERVICE_USER : ${SERVICE_USER} +EOF + info_searx + [[ -n ${1} ]] && err_msg "$1" +} + +info_searx() { + # shellcheck disable=SC1117 + cat <<EOF + +To activate morty in searx, add result_proxy to your settings.yml:: + + result_proxy: + url : ${PUBLIC_URL_MORTY}/ + +further read: ${DOCS_URL}/admin/morty.html + +EOF +} + +main() { + rst_title "$SERVICE_NAME" part + + required_commands \ + dpkg apt-get install git wget curl \ + || exit + + local _usage="ERROR: unknown or missing $1 command $2" + + case $1 in + --source-only) ;; + -h|--help) usage; exit 0;; + + shell) + sudo_or_exit + interactive_shell "${SERVICE_USER}" + ;; + inspect) + case $2 in + service) + sudo_or_exit + inspect_service + ;; + *) usage "$_usage"; exit 42;; + esac ;; + install) + sudo_or_exit + case $2 in + all) install_all ;; + user) assert_user ;; + *) usage "$_usage"; exit 42;; + esac ;; + update) + sudo_or_exit + case $2 in + morty) update_morty ;; + *) usage "$_usage"; exit 42;; + esac ;; + remove) + sudo_or_exit + case $2 in + all) remove_all;; + user) drop_service_account "${SERVICE_USER}" ;; + *) usage "$_usage"; exit 42;; + esac ;; + activate) + sudo_or_exit + case $2 in + service) systemd_activate_service "${SERVICE_NAME}" ;; + *) usage "$_usage"; exit 42;; + esac ;; + deactivate) + sudo_or_exit + case $2 in + service) systemd_deactivate_service "${SERVICE_NAME}" ;; + *) usage "$_usage"; exit 42;; + esac ;; + apache) + sudo_or_exit + case $2 in + install) install_apache_site ;; + remove) remove_apache_site ;; + *) usage "$_usage"; exit 42;; + esac ;; + info) + case $2 in + searx) info_searx ;; + *) usage "$_usage"; exit 42;; + esac ;; + option) + sudo_or_exit + case $2 in + debug-on) enable_debug ;; + debug-off) disable_debug ;; + *) usage "$_usage"; exit 42;; + esac ;; + + *) usage "ERROR: unknown or missing command $1"; exit 42;; + esac +} + +install_all() { + rst_title "Install $SERVICE_NAME (service)" + assert_user + wait_key + install_go "${GO_PKG_URL}" "${GO_TAR}" "${SERVICE_USER}" + wait_key + install_morty + wait_key + systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" + wait_key + info_searx + if ! service_is_available "http://${MORTY_LISTEN}" ; then + err_msg "Morty does not listening on: http://${MORTY_LISTEN}" + fi + if apache_is_installed; then + info_msg "Apache is installed on this host." + if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then + install_apache_site + fi + fi + if ask_yn "Do you want to inspect the installation?" Yn; then + inspect_service + fi + +} + +remove_all() { + rst_title "De-Install $SERVICE_NAME (service)" + + rst_para "\ +It goes without saying that this script can only be used to remove +installations that were installed with this script." + + if systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then + drop_service_account "${SERVICE_USER}" + fi +} + +assert_user() { + rst_title "user $SERVICE_USER" section + echo + tee_stderr 1 <<EOF | bash | prefix_stdout +sudo -H adduser --shell /bin/bash --system --home $SERVICE_HOME \ + --disabled-password --group --gecos 'Morty' $SERVICE_USER +sudo -H usermod -a -G shadow $SERVICE_USER +groups $SERVICE_USER +EOF + SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)" + export SERVICE_HOME + echo "export SERVICE_HOME=$SERVICE_HOME" + + cat > "$GO_ENV" <<EOF +export GOPATH=\$HOME/go-apps +export PATH=\$PATH:\$HOME/local/go/bin:\$GOPATH/bin +EOF + echo "Environment $GO_ENV has been setup." + + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" +grep -qFs -- 'source $GO_ENV' ~/.profile || echo 'source $GO_ENV' >> ~/.profile +EOF +} + +morty_is_installed() { + [[ -f $SERVICE_HOME/go-apps/bin/morty ]] +} + +_svcpr=" |${SERVICE_USER}| " + +install_morty() { + rst_title "Install morty in user's ~/go-apps" section + echo + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr" +go get -v -u github.com/asciimoo/morty +EOF + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr" +cd \$GOPATH/src/github.com/asciimoo/morty +go test +go test -benchmem -bench . +EOF +} + +update_morty() { + rst_title "Update morty" section + echo + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr" +go get -v -u github.com/asciimoo/morty +EOF + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr" +cd \$GOPATH/src/github.com/asciimoo/morty +go test +go test -benchmem -bench . +EOF +} + +set_service_env_debug() { + + # usage: set_service_env_debug [false|true] + + # shellcheck disable=SC2034 + local SERVICE_ENV_DEBUG="${1:-false}" + if systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then + systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" + fi +} + +inspect_service() { + + rst_title "service status & log" + + cat <<EOF + +sourced ${DOT_CONFIG#"$REPO_ROOT/"} : + + MORTY_LISTEN : ${MORTY_LISTEN} + +EOF + + if service_account_is_available "$SERVICE_USER"; then + info_msg "service account $SERVICE_USER available." + else + err_msg "service account $SERVICE_USER not available!" + fi + if go_is_available "$SERVICE_USER"; then + info_msg "~$SERVICE_USER: go is installed" + else + err_msg "~$SERVICE_USER: go is not installed" + fi + if morty_is_installed; then + info_msg "~$SERVICE_USER: morty app is installed" + else + err_msg "~$SERVICE_USER: morty app is not installed!" + fi + + if ! service_is_available "http://${MORTY_LISTEN}" ; then + err_msg "Morty does not listening on: http://${MORTY_LISTEN}" + echo -e "${_Green}stop with [${_BCyan}CTRL-C${_Green}] or .." + wait_key + fi + + local _debug_on + if ask_yn "Enable filtron debug mode?"; then + enable_debug + _debug_on=1 + fi + + echo + systemctl --no-pager -l status "${SERVICE_NAME}" + echo + + info_msg "morty URL --> http://${MORTY_LISTEN}" + info_msg "public URL --> ${PUBLIC_URL_MORTY}" + # shellcheck disable=SC2059 + printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log" + read -r -s -n1 -t 2 + echo + while true; do + trap break 2 + journalctl -f -u "${SERVICE_NAME}" + done + + if [[ $_debug_on == 1 ]]; then + FORCE_SELECTION=Y disable_debug + fi + return 0 +} + + +enable_debug() { + warn_msg "Do not enable debug in production enviroments!!" + info_msg "Enabling debug option needs to reinstall systemd service!" + set_service_env_debug true +} + +disable_debug() { + info_msg "Disabling debug option needs to reinstall systemd service!" + set_service_env_debug false +} + +install_apache_site() { + + rst_title "Install Apache site $APACHE_MORTY_SITE" + + rst_para "\ +This installs a reverse proxy (ProxyPass) into apache site (${APACHE_MORTY_SITE})" + + ! apache_is_installed && err_msg "Apache is not installed." + + if ! ask_yn "Do you really want to continue?"; then + return + fi + + a2enmod headers + a2enmod proxy + a2enmod proxy_http + + echo + apache_install_site "${APACHE_MORTY_SITE}" + + info_msg "testing public url .." + if ! service_is_available "${PUBLIC_URL_MORTY}"; then + err_msg "Public service at ${PUBLIC_URL_MORTY} is not available!" + fi +} + +remove_apache_site() { + + rst_title "Remove Apache site $APACHE_MORTY_SITE" + + rst_para "\ +This removes apache site ${APACHE_MORTY_SITE}." + + ! apache_is_installed && err_msg "Apache is not installed." + + if ! ask_yn "Do you really want to continue?"; then + return + fi + + apache_remove_site "$APACHE_MORTY_SITE" +} +# ---------------------------------------------------------------------------- +main "$@" +# ---------------------------------------------------------------------------- diff --git a/utils/searx.sh b/utils/searx.sh new file mode 100755 index 000000000..942aa07ce --- /dev/null +++ b/utils/searx.sh @@ -0,0 +1,589 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- +# SPDX-License-Identifier: AGPL-3.0-or-later +# shellcheck disable=SC2001 + +# shellcheck source=utils/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +source_dot_config + +# ---------------------------------------------------------------------------- +# config +# ---------------------------------------------------------------------------- + +SEARX_INTERNAL_URL="${SEARX_INTERNAL_URL:-127.0.0.1:8888}" + +SEARX_URL_PATH="${SEARX_URL_PATH:-$(echo "${PUBLIC_URL}" \ +| sed -e 's,^.*://[^/]*\(/.*\),\1,g')}" +[[ "${SEARX_URL_PATH}" == "${PUBLIC_URL}" ]] && SEARX_URL_PATH=/ +SEARX_INSTANCE_NAME="${SEARX_INSTANCE_NAME:-searx@$(echo "$PUBLIC_URL" \ +| sed -e 's,^.*://\([^\:/]*\).*,\1,g') }" + +SERVICE_NAME="searx" +SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}" +SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}" +SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}" +# shellcheck disable=SC2034 +SERVICE_GROUP="${SERVICE_USER}" + +SEARX_GIT_URL="${SEARX_GIT_URL:-https://github.com/asciimoo/searx.git}" +SEARX_GIT_BRANCH="${SEARX_GIT_BRANCH:-master}" +SEARX_PYENV="${SERVICE_HOME}/searx-pyenv" +SEARX_SRC="${SERVICE_HOME}/searx-src" +SEARX_SETTINGS="${SEARX_SRC}/searx/settings.yml" +SEARX_UWSGI_APP="searx.ini" +# shellcheck disable=SC2034 +SEARX_UWSGI_SOCKET="/run/uwsgi/app/searx/socket" + +# FIXME: Arch Linux & RHEL should be added + +SEARX_APT_PACKAGES="\ + uwsgi uwsgi-plugin-python3 \ + git build-essential \ + libxslt-dev python3-dev python3-babel python3-venv \ + zlib1g-dev libffi-dev libssl-dev \ +" + +# Apache Settings + +APACHE_APT_PACKAGES="\ + libapache2-mod-uwsgi \ +" + +APACHE_SEARX_SITE="searx.conf" + +# shellcheck disable=SC2034 +CONFIG_FILES=( + "${uWSGI_SETUP}/apps-available/${SEARX_UWSGI_APP}" +) + +# shellcheck disable=SC2034 +CONFIG_BACKUP_ENCRYPTED=( + "${SEARX_SETTINGS}" +) + +# ---------------------------------------------------------------------------- +usage() { +# ---------------------------------------------------------------------------- + + # shellcheck disable=SC1117 + cat <<EOF + +usage:: + + $(basename "$0") shell + $(basename "$0") install [all|user|pyenv|searx-src|apache] + $(basename "$0") update [searx] + $(basename "$0") remove [all|user|pyenv|searx-src] + $(basename "$0") activate [service] + $(basename "$0") deactivate [service] + $(basename "$0") inspect [service] + $(basename "$0") option [debug-on|debug-off] + $(basename "$0") apache [install|remove] + +shell + start interactive shell from user ${SERVICE_USER} +install / remove + :all: complete (de-) installation of searx service + :user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME) + :searx-src: clone $SEARX_GIT_URL + :pyenv: create/remove virtualenv (python) in $SEARX_PYENV +update searx + Update searx installation ($SERVICE_HOME) +activate service + activate and start service daemon (systemd unit) +deactivate service + stop and deactivate service daemon (systemd unit) +inspect service + run some small tests and inspect service's status and log +option + set one of the available options +apache + :install: apache site with the searx uwsgi app + :remove: apache site ${APACHE_FILTRON_SITE} + +If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/"}' file:: + + PUBLIC_URL : ${PUBLIC_URL} + PUBLIC_HOST : ${PUBLIC_HOST} + SEARX_INSTANCE_NAME : ${SEARX_INSTANCE_NAME} + SERVICE_USER : ${SERVICE_USER} + +EOF + [[ -n ${1} ]] && err_msg "$1" +} + +main() { + rst_title "$SEARX_INSTANCE_NAME" part + + required_commands \ + dpkg systemctl apt-get install git wget curl \ + || exit + + local _usage="unknown or missing $1 command $2" + + case $1 in + --source-only) ;; + -h|--help) usage; exit 0;; + + shell) + sudo_or_exit + interactive_shell "${SERVICE_USER}" + ;; + inspect) + case $2 in + service) + sudo_or_exit + inspect_service + ;; + *) usage "$_usage"; exit 42;; + esac ;; + install) + sudo_or_exit + case $2 in + all) install_all ;; + user) assert_user ;; + pyenv) create_pyenv ;; + searx-src) clone_searx ;; + *) usage "$_usage"; exit 42;; + esac ;; + update) + sudo_or_exit + case $2 in + searx) update_searx;; + *) usage "$_usage"; exit 42;; + esac ;; + remove) + sudo_or_exit + case $2 in + all) remove_all;; + user) drop_service_account "${SERVICE_USER}";; + pyenv) remove_pyenv ;; + searx-src) remove_searx ;; + *) usage "$_usage"; exit 42;; + esac ;; + activate) + sudo_or_exit + case $2 in + service) + activate_service ;; + *) usage "$_usage"; exit 42;; + esac ;; + deactivate) + sudo_or_exit + case $2 in + service) deactivate_service ;; + *) usage "$_usage"; exit 42;; + esac ;; + option) + sudo_or_exit + case $2 in + debug-on) echo; enable_debug ;; + debug-off) echo; disable_debug ;; + *) usage "$_usage"; exit 42;; + esac ;; + apache) + sudo_or_exit + case $2 in + install) install_apache_site ;; + remove) remove_apache_site ;; + *) usage "$_usage"; exit 42;; + esac ;; + + *) usage "unknown or missing command $1"; exit 42;; + esac +} + +_service_prefix=" |$SERVICE_USER| " + +install_all() { + rst_title "Install $SEARX_INSTANCE_NAME (service)" + pkg_install "$SEARX_APT_PACKAGES" + wait_key + assert_user + wait_key + clone_searx + wait_key + create_pyenv + wait_key + configure_searx + wait_key + test_local_searx + wait_key + install_searx_uwsgi + if ! service_is_available "http://$SEARX_INTERNAL_URL"; then + err_msg "URL http://$SEARX_INTERNAL_URL not available, check searx & uwsgi setup!" + fi + if ask_yn "Do you want to inspect the installation?" Yn; then + inspect_service + fi + +} + +update_searx() { + rst_title "Update searx instance" + + echo + tee_stderr 0.3 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +cp -f ${SEARX_SETTINGS} ${SEARX_SETTINGS}.backup +git stash push -m "BACKUP -- 'update server' at ($(date))" +git checkout -B "$SEARX_GIT_BRANCH" +git pull +cd ${SEARX_SRC} +${SEARX_SRC}/manage.sh update_packages +EOF + configure_searx + + rst_title "${SEARX_SETTINGS}" section + rst_para 'Diff between new setting file (<) and backup (>):' + echo + $DIFF_CMD "${SEARX_SETTINGS}" "${SEARX_SETTINGS}.backup" + + local action + choose_one action "What should happen to the settings file? " \ + "keep new configuration" \ + "revert to the old configuration (backup file)" \ + "start interactiv shell" + case $action in + "keep new configuration") + info_msg "continue using new settings file" + ;; + "revert to the old configuration (backup file)") + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +cp -f ${SEARX_SETTINGS}.backup ${SEARX_SETTINGS} +EOF + ;; + "start interactiv shell") + interactive_shell "${SERVICE_USER}" + ;; + esac + chown "${SERVICE_USER}:${SERVICE_USER}" "${SEARX_SETTINGS}" + + # shellcheck disable=SC2016 + rst_para 'Diff between local modified settings (<) and $SEARX_GIT_BRANCH branch (>):' + echo + git_diff + wait_key + uWSGI_restart +} + +remove_all() { + rst_title "De-Install $SEARX_INSTANCE_NAME (service)" + + rst_para "\ +It goes without saying that this script can only be used to remove +installations that were installed with this script." + + if ! ask_yn "Do you really want to deinstall $SEARX_INSTANCE_NAME?"; then + return + fi + remove_searx_uwsgi + wait_key + drop_service_account "${SERVICE_USER}" + if service_is_available "${PUBLIC_URL}"; then + MSG="** Don't forgett to remove your public site! (${PUBLIC_URL}) **" wait_key 10 + fi +} + +assert_user() { + rst_title "user $SERVICE_USER" section + echo + tee_stderr 1 <<EOF | bash | prefix_stdout +sudo -H adduser --shell /bin/bash --system --home "$SERVICE_HOME" \ + --disabled-password --group --gecos 'searx' $SERVICE_USER +sudo -H usermod -a -G shadow $SERVICE_USER +groups $SERVICE_USER +EOF + #SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)" + #export SERVICE_HOME + #echo "export SERVICE_HOME=$SERVICE_HOME" +} + +clone_is_available() { + [[ -f "$SEARX_SETTINGS" ]] +} + +# shellcheck disable=SC2164 +clone_searx() { + rst_title "Clone searx sources" section + echo + SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME 2>/dev/null)" + if [[ ! "${SERVICE_HOME}" ]]; then + err_msg "to clone searx sources, user $SERVICE_USER hast to be created first" + return 42 + fi + export SERVICE_HOME + + git_clone "$SEARX_GIT_URL" "$SEARX_SRC" \ + "$SEARX_GIT_BRANCH" "$SERVICE_USER" + + pushd "${SEARX_SRC}" > /dev/null + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +cd "${SEARX_SRC}" +git config user.email "$ADMIN_EMAIL" +git config user.name "$ADMIN_NAME" +git config --list +EOF + popd > /dev/null +} + +remove_searx() { + rst_title "Drop searx sources" section + if ask_yn "Do you really want to drop searx sources ($SEARX_SRC)?"; then + rm -rf "$SEARX_SRC" + else + rst_para "Leave searx sources unchanged." + fi +} + +pyenv_is_available() { + [[ -f "${SEARX_PYENV}/bin/activate" ]] +} + +create_pyenv() { + rst_title "Create virtualenv (python)" section + echo + if [[ ! -f "${SEARX_SRC}/manage.sh" ]]; then + err_msg "to create pyenv for searx, searx has to be cloned first" + return 42 + fi + info_msg "create pyenv in ${SEARX_PYENV}" + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +rm -rf "${SEARX_PYENV}" +python3 -m venv "${SEARX_PYENV}" +grep -qFs -- 'source ${SEARX_PYENV}/bin/activate' ~/.profile \ + || echo 'source ${SEARX_PYENV}/bin/activate' >> ~/.profile +EOF + info_msg "inspect python's virtual environment" + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +command -v python && python --version +EOF + wait_key + info_msg "install needed python packages" + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +${SEARX_SRC}/manage.sh update_packages +EOF +} + +remove_pyenv() { + rst_title "Remove virtualenv (python)" section + if ! ask_yn "Do you really want to drop ${SEARX_PYENV} ?"; then + return + fi + info_msg "remove pyenv activation from ~/.profile" + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +grep -v 'source ${SEARX_PYENV}/bin/activate' ~/.profile > ~/.profile.## +mv ~/.profile.## ~/.profile +EOF + rm -rf "${SEARX_PYENV}" +} + +configure_searx() { + rst_title "Configure searx" section + rst_para "Setup searx config located at $SEARX_SETTINGS" + echo + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +sed -i -e "s/ultrasecretkey/$(openssl rand -hex 16)/g" "$SEARX_SETTINGS" +sed -i -e "s/{instance_name}/${SEARX_INSTANCE_NAME}/g" "$SEARX_SETTINGS" +EOF +} + +test_local_searx() { + rst_title "Testing searx instance localy" section + echo + + if service_is_available "http://$SEARX_INTERNAL_URL" &>/dev/null; then + err_msg "URL/port http://$SEARX_INTERNAL_URL is already in use, you" + err_msg "should stop that service before starting local tests!" + if ! ask_yn "Continue with local tests?"; then + return + fi + fi + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +sed -i -e "s/debug : False/debug : True/g" "$SEARX_SETTINGS" +timeout 10 python3 searx/webapp.py & +sleep 3 +curl --location --verbose --head --insecure $SEARX_INTERNAL_URL +sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS" +EOF +} + +install_searx_uwsgi() { + rst_title "Install searx's uWSGI app (searx.ini)" section + echo + uWSGI_install_app "$SEARX_UWSGI_APP" +} + +remove_searx_uwsgi() { + rst_title "Remove searx's uWSGI app (searx.ini)" section + echo + uWSGI_remove_app "$SEARX_UWSGI_APP" +} + +activate_service() { + rst_title "Activate $SEARX_INSTANCE_NAME (service)" section + echo + uWSGI_enable_app "$SEARX_UWSGI_APP" + uWSGI_restart +} + +deactivate_service() { + rst_title "De-Activate $SEARX_INSTANCE_NAME (service)" section + echo + uWSGI_disable_app "$SEARX_UWSGI_APP" + uWSGI_restart +} + +git_diff() { + sudo -H -u "${SERVICE_USER}" -i <<EOF +cd ${SEARX_SRC} +git --no-pager diff +EOF +} + +enable_debug() { + warn_msg "Do not enable debug in production enviroments!!" + info_msg "try to enable debug mode ..." + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +sed -i -e "s/debug : False/debug : True/g" "$SEARX_SETTINGS" +EOF + uWSGI_restart +} + +disable_debug() { + info_msg "try to disable debug mode ..." + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS" +EOF + uWSGI_restart +} + +inspect_service() { + rst_title "service status & log" + cat <<EOF + +sourced ${DOT_CONFIG#"$REPO_ROOT/"} : + + PUBLIC_URL : ${PUBLIC_URL} + PUBLIC_HOST : ${PUBLIC_HOST} + SEARX_URL_PATH : ${SEARX_URL_PATH} + SEARX_INSTANCE_NAME : ${SEARX_INSTANCE_NAME} + SEARX_INTERNAL_URL : ${SEARX_INTERNAL_URL} + +EOF + + apache_is_installed && info_msg "Apache is installed." + + if service_account_is_available "$SERVICE_USER"; then + info_msg "Service account $SERVICE_USER exists." + else + err_msg "Service account $SERVICE_USER does not exists!" + fi + + if pyenv_is_available; then + info_msg "~$SERVICE_USER: python environment is available." + else + err_msg "~$SERVICE_USER: python environment is not available!" + fi + + if clone_is_available; then + info_msg "~$SERVICE_USER: Searx software is installed." + else + err_msg "~$SERVICE_USER: Missing searx software!" + fi + + if uWSGI_app_enabled "$SEARX_UWSGI_APP"; then + info_msg "uWSGI app $SEARX_UWSGI_APP is enabled." + else + err_msg "uWSGI app $SEARX_UWSGI_APP not enabled!" + fi + + uWSGI_app_available "$SEARX_UWSGI_APP" \ + || err_msg "uWSGI app $SEARX_UWSGI_APP not available!" + + if ! service_is_available "http://${SEARX_INTERNAL_URL}"; then + err_msg "uWSGI app (service) at http://${SEARX_INTERNAL_URL} is not available!" + echo -e "${_Green}stop with [${_BCyan}CTRL-C${_Green}] or .." + wait_key + fi + + if ! service_is_available "${PUBLIC_URL}"; then + err_msg "Public service at ${PUBLIC_URL} is not available!" + fi + + local _debug_on + if ask_yn "Enable searx debug mode?"; then + enable_debug + _debug_on=1 + fi + echo + systemctl --no-pager -l status "${SERVICE_NAME}" + echo + + info_msg "public URL --> ${PUBLIC_URL}" + info_msg "internal URL --> http://${SEARX_INTERNAL_URL}" + # shellcheck disable=SC2059 + printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log" + read -r -s -n1 -t 2 + echo + while true; do + trap break 2 + #journalctl -f -u "${SERVICE_NAME}" + tail -f /var/log/uwsgi/app/searx.log + done + + if [[ $_debug_on == 1 ]]; then + disable_debug + fi + return 0 +} + +install_apache_site() { + rst_title "Install Apache site $APACHE_SEARX_SITE" + + rst_para "\ +This installs the searx uwsgi app as apache site. If your server ist public to +the internet you should instead use a reverse proxy (filtron) to block +excessively bot queries." + + ! apache_is_installed && err_msg "Apache is not installed." + + if ! ask_yn "Do you really want to install apache site for searx-uwsgi?"; then + return + fi + + pkg_install "$APACHE_APT_PACKAGES" + a2enmod uwsgi + + echo + apache_install_site --variant=uwsgi "${APACHE_SEARX_SITE}" + + if ! service_is_available "${PUBLIC_URL}"; then + err_msg "Public service at ${PUBLIC_URL} is not available!" + fi +} + +remove_apache_site() { + + rst_title "Remove Apache site ${APACHE_SEARX_SITE}" + + rst_para "\ +This removes apache site ${APACHE_SEARX_SITE}." + + ! apache_is_installed && err_msg "Apache is not installed." + + if ! ask_yn "Do you really want to continue?"; then + return + fi + + apache_remove_site "${APACHE_SEARX_SITE}" +} + +# ---------------------------------------------------------------------------- +main "$@" +# ---------------------------------------------------------------------------- diff --git a/utils/templates/etc/apache2/sites-available/morty.conf b/utils/templates/etc/apache2/sites-available/morty.conf new file mode 100644 index 000000000..6bcc77b78 --- /dev/null +++ b/utils/templates/etc/apache2/sites-available/morty.conf @@ -0,0 +1,28 @@ +# -*- coding: utf-8; mode: apache -*- + +ProxyPreserveHost On + +<Location ${PUBLIC_URL_PATH_MORTY} > + + <IfModule mod_security2.c> + SecRuleEngine Off + </IfModule> + + Require all granted + + Order deny,allow + Deny from all + #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + ProxyPass http://${MORTY_LISTEN} + RequestHeader set X-Script-Name ${PUBLIC_URL_PATH_MORTY} + + # In Apache it seems, that setting HTTP_HOST header direct here does have no + # effect. I needed to set 'ProxyPreserveHost On' (see above). HTTP_HOST is + # needed by searx to render correct *Search URL* in the *Link* box and + # *saved preference*. + + # RequestHeader set Host ${PUBLIC_URL_PATH_MORTY} + +</Location> diff --git a/utils/templates/etc/apache2/sites-available/searx.conf:filtron b/utils/templates/etc/apache2/sites-available/searx.conf:filtron new file mode 100644 index 000000000..5ede66301 --- /dev/null +++ b/utils/templates/etc/apache2/sites-available/searx.conf:filtron @@ -0,0 +1,33 @@ +# -*- coding: utf-8; mode: apache -*- + +ProxyPreserveHost On + +# SecRuleRemoveById 981054 +# SecRuleRemoveById 981059 +# SecRuleRemoveById 981060 +# SecRuleRemoveById 950907 + +<Location ${FILTRON_URL_PATH} > + + <IfModule mod_security2.c> + SecRuleEngine Off + </IfModule> + + Require all granted + + Order deny,allow + Deny from all + #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + ProxyPass http://${FILTRON_LISTEN} + RequestHeader set X-Script-Name ${FILTRON_URL_PATH} + + # In Apache it seems, that setting HTTP_HOST header direct here does have no + # effect. I needed to set 'ProxyPreserveHost On' (see above). HTTP_HOST is + # needed by searx to render correct *Search URL* in the *Link* box and + # *saved preference*. + + # RequestHeader set Host ${PUBLIC_HOST} + +</Location> diff --git a/utils/templates/etc/apache2/sites-available/searx.conf:uwsgi b/utils/templates/etc/apache2/sites-available/searx.conf:uwsgi new file mode 100644 index 000000000..21e01ac4e --- /dev/null +++ b/utils/templates/etc/apache2/sites-available/searx.conf:uwsgi @@ -0,0 +1,27 @@ +# -*- coding: utf-8; mode: apache -*- + +<IfModule mod_uwsgi.c> + + # SetEnvIf Request_URI "${SEARX_URL_PATH}" dontlog + # CustomLog /dev/null combined env=dontlog + + <Location ${SEARX_URL_PATH}> + + <IfModule mod_security2.c> + SecRuleEngine Off + </IfModule> + + Require all granted + + Options FollowSymLinks Indexes + SetHandler uwsgi-handler + uWSGISocket ${SEARX_UWSGI_SOCKET} + + Order deny,allow + Deny from all + # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + </Location> + +</IfModule> diff --git a/utils/templates/etc/filtron/rules.json b/utils/templates/etc/filtron/rules.json new file mode 100644 index 000000000..f4a99b3e8 --- /dev/null +++ b/utils/templates/etc/filtron/rules.json @@ -0,0 +1,125 @@ +[ + { + "name": "roboagent limit", + "filters": [ + "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client|Ruby|UniversalFeedParser)" + ], + "limit": 0, + "stop": true, + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "botlimit", + "filters": [ + "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)" + ], + "limit": 0, + "stop": true, + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { "name": "suspiciously frequent IP", + "filters": [], + "interval": 600, + "limit": 30, + "aggregations": [ + "Header:X-Forwarded-For" + ], + "actions":[ + {"name":"log"} + ] + }, + { "name": "search request", + "filters": [ + "Param:q", + "Path=^(/|/search)$" + ], + "interval": 61, + "limit": 999, + "subrules": [ + { + "name": "missing Accept-Language", + "filters": ["!Header:Accept-Language"], + "limit": 0, + "stop": true, + "actions": [ + {"name": "block", + "params": {"message": "Rate limit exceeded"}} + ] + }, + { + "name": "suspiciously Connection=close header", + "filters": ["Header:Connection=close"], + "limit": 0, + "stop": true, + "actions": [ + {"name": "block", + "params": {"message": "Rate limit exceeded"}} + ] + }, + { + "name": "IP limit", + "interval": 61, + "limit": 9, + "stop": true, + "aggregations": [ + "Header:X-Forwarded-For" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "rss/json limit", + "filters": [ + "Param:format=(csv|json|rss)" + ], + "interval": 121, + "limit": 2, + "stop": true, + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "useragent limit", + "interval": 61, + "limit": 199, + "aggregations": [ + "Header:User-Agent" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + } + ] + } +] diff --git a/utils/templates/etc/uwsgi/apps-available/searx.ini b/utils/templates/etc/uwsgi/apps-available/searx.ini new file mode 100644 index 000000000..cff47f896 --- /dev/null +++ b/utils/templates/etc/uwsgi/apps-available/searx.ini @@ -0,0 +1,62 @@ +[uwsgi] + +# uWSGI core +# ---------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core + +# Who will run the code +uid = ${SERVICE_USER} +gid = ${SERVICE_GROUP} + +# chdir to specified directory before apps loading +chdir = ${SEARX_SRC}/searx + +# disable logging for privacy +disable-logging = true + +# The right granted on the created socket +chmod-socket = 666 + +# Plugin to use and interpretor config +single-interpreter = true + +# enable master process +master = true + +# load apps in each worker instead of the master +lazy-apps = true + +# load uWSGI plugins +plugin = python3,http + +# By default the Python plugin does not initialize the GIL. This means your +# app-generated threads will not run. If you need threads, remember to enable +# them with enable-threads. Running uWSGI in multithreading mode (with the +# threads options) will automatically enable threading support. This *strange* +# default behaviour is for performance reasons. +enable-threads = true + + +# plugin: python +# -------------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python + +# load a WSGI module +module = searx.webapp + +# set PYTHONHOME/virtualenv +virtualenv = ${SEARX_PYENV} + +# add directory (or glob) to pythonpath +pythonpath = ${SEARX_SRC} + + +# plugin http +# ----------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http + +# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html +http = ${SEARX_INTERNAL_URL} diff --git a/utils/templates/lib/systemd/system/filtron.service b/utils/templates/lib/systemd/system/filtron.service new file mode 100644 index 000000000..3b0c6edcc --- /dev/null +++ b/utils/templates/lib/systemd/system/filtron.service @@ -0,0 +1,29 @@ +[Unit] + +Description=${SERVICE_NAME} +After=syslog.target +After=network.target + +[Service] + +Type=simple +User=${SERVICE_USER} +Group=${SERVICE_GROUP} +WorkingDirectory=${SERVICE_HOME} +ExecStart=${SERVICE_HOME}/go-apps/bin/filtron -api '${FILTRON_API}' -listen '${FILTRON_LISTEN}' -rules '${FILTRON_RULES}' -target '${FILTRON_TARGET}' + +Restart=always +Environment=USER=${SERVICE_USER} HOME=${SERVICE_HOME} + +# Some distributions may not support these hardening directives. If you cannot +# start the service due to an unknown option, comment out the ones not supported +# by your version of systemd. + +ProtectSystem=full +PrivateDevices=yes +PrivateTmp=yes +NoNewPrivileges=true + +[Install] + +WantedBy=multi-user.target diff --git a/utils/templates/lib/systemd/system/morty.service b/utils/templates/lib/systemd/system/morty.service new file mode 100644 index 000000000..d463c5097 --- /dev/null +++ b/utils/templates/lib/systemd/system/morty.service @@ -0,0 +1,29 @@ +[Unit] + +Description=${SERVICE_NAME} +After=syslog.target +After=network.target + +[Service] + +Type=simple +User=${SERVICE_USER} +Group=${SERVICE_GROUP} +WorkingDirectory=${SERVICE_HOME} +ExecStart=${SERVICE_HOME}/go-apps/bin/morty -key '' -listen '${MORTY_LISTEN}' -timeout ${MORTY_TIMEOUT} + +Restart=always +Environment=USER=${SERVICE_USER} HOME=${SERVICE_HOME} DEBUG=${SERVICE_ENV_DEBUG} + +# Some distributions may not support these hardening directives. If you cannot +# start the service due to an unknown option, comment out the ones not supported +# by your version of systemd. + +ProtectSystem=full +PrivateDevices=yes +PrivateTmp=yes +NoNewPrivileges=true + +[Install] + +WantedBy=multi-user.target |