diff options
author | Markus Heiser <markus.heiser@darmarit.de> | 2022-05-27 18:43:14 +0200 |
---|---|---|
committer | Markus Heiser <markus.heiser@darmarit.de> | 2022-07-30 13:39:35 +0200 |
commit | 782f73540e2d383ea122716507ccd9582918ab51 (patch) | |
tree | 2cca6c47e63cdf383883a2a47ac8329e629bb40d /utils | |
parent | 81bba4486916dbca4467a7e8c8b839a7c55c2a09 (diff) | |
download | searxng-782f73540e2d383ea122716507ccd9582918ab51.tar.gz searxng-782f73540e2d383ea122716507ccd9582918ab51.zip |
[utils/searxng.sh] implement new script to install SearXNG
Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
Diffstat (limited to 'utils')
-rwxr-xr-x | utils/lib.sh | 47 | ||||
-rwxr-xr-x | utils/lib_install.sh | 7 | ||||
-rwxr-xr-x | utils/lib_redis.sh | 27 | ||||
-rw-r--r-- | utils/lxc-searxng.env (renamed from utils/lxc-searx.env) | 51 | ||||
-rwxr-xr-x | utils/lxc.sh | 2 | ||||
-rwxr-xr-x | utils/morty.sh | 2 | ||||
-rwxr-xr-x | utils/searxng.sh | 894 | ||||
-rw-r--r-- | utils/searxng_check.py | 7 | ||||
-rw-r--r-- | utils/templates/etc/httpd/sites-available/searxng.conf | 41 | ||||
-rw-r--r-- | utils/templates/etc/httpd/sites-available/searxng.conf:socket | 41 | ||||
-rw-r--r-- | utils/templates/etc/nginx/default.apps-available/searxng.conf | 29 | ||||
-rw-r--r-- | utils/templates/etc/nginx/default.apps-available/searxng.conf:socket | 26 | ||||
-rw-r--r-- | utils/templates/etc/searxng/settings.yml | 78 | ||||
-rw-r--r-- | utils/templates/etc/uwsgi/apps-archlinux/searxng.ini | 18 | ||||
-rw-r--r-- | utils/templates/etc/uwsgi/apps-archlinux/searxng.ini:socket | 17 | ||||
-rw-r--r-- | utils/templates/etc/uwsgi/apps-available/searxng.ini | 24 | ||||
-rw-r--r-- | utils/templates/etc/uwsgi/apps-available/searxng.ini:socket | 23 |
17 files changed, 1147 insertions, 187 deletions
diff --git a/utils/lib.sh b/utils/lib.sh index 257c15024..32db47524 100755 --- a/utils/lib.sh +++ b/utils/lib.sh @@ -195,7 +195,7 @@ wait_key(){ [[ -n $_t ]] && _t="-t $_t" printf "$msg" # shellcheck disable=SC2086 - read -r -s -n1 $_t + read -r -s -n1 $_t || true echo clean_stdin } @@ -1027,7 +1027,7 @@ nginx_include_apps_enabled() { local include_directive="include ${NGINX_APPS_ENABLED}/*.conf;" local include_directive_re="^\s*include ${NGINX_APPS_ENABLED}/\*\.conf;" - info_msg "checking existence: '${include_directive}' in file ${server_conf}" + info_msg "checking existence: '${include_directive}' in file ${server_conf}" if grep "${include_directive_re}" "${server_conf}"; then info_msg "OK, already exists." return @@ -1117,7 +1117,7 @@ apache_distro_setup() { APACHE_SITES_AVAILABLE="/etc/httpd/sites-available" APACHE_SITES_ENABLED="/etc/httpd/sites-enabled" APACHE_MODULES="modules" - APACHE_PACKAGES="httpd" + APACHE_PACKAGES="httpd mod_ssl" ;; *) err_msg "$DIST_ID-$DIST_VERS: apache not yet implemented" @@ -1249,8 +1249,6 @@ apache_dissable_site() { # ----- uWSGI_SETUP="${uWSGI_SETUP:=/etc/uwsgi}" -uWSGI_USER= -uWSGI_GROUP= # How distros manage uWSGI apps is very different. From uWSGI POV read: # - https://uwsgi-docs.readthedocs.io/en/latest/Management.html @@ -1276,13 +1274,14 @@ uWSGI_distro_setup() { ;; fedora-*|centos-7) # systemd --> /usr/lib/systemd/system/uwsgi.service - # The unit file starts uWSGI in emperor mode (/etc/uwsgi.ini), see - # - https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html + # Fedora runs uWSGI in emperor-tyrant mode: in Tyrant mode the + # Emperor will run the vassal using the UID/GID of the vassal + # configuration file [1] (user and group of the app .ini file). + # There are some quirks abbout additional POSIX groups in uWSGI + # 2.0.x, read at least: https://github.com/unbit/uwsgi/issues/2099 uWSGI_APPS_AVAILABLE="${uWSGI_SETUP}/apps-available" uWSGI_APPS_ENABLED="${uWSGI_SETUP}.d" uWSGI_PACKAGES="uwsgi" - uWSGI_USER="uwsgi" - uWSGI_GROUP="uwsgi" ;; *) err_msg "$DIST_ID-$DIST_VERS: uWSGI not yet implemented" @@ -1344,30 +1343,6 @@ uWSGI_restart() { esac } -uWSGI_prepare_app() { - - # usage: uWSGI_prepare_app <myapp.ini> - - [[ -z $1 ]] && die_caller 42 "missing argument <myapp.ini>" - - local APP="${1%.*}" - - case $DIST_ID-$DIST_VERS in - fedora-*|centos-7) - # in emperor mode, the uwsgi user is the owner of the sockets - info_msg "prepare (uwsgi:uwsgi) /run/uwsgi/app/${APP}" - mkdir -p "/run/uwsgi/app/${APP}" - chown -R "uwsgi:uwsgi" "/run/uwsgi/app/${APP}" - ;; - *) - info_msg "prepare (${SERVICE_USER}:${SERVICE_GROUP}) /run/uwsgi/app/${APP}" - mkdir -p "/run/uwsgi/app/${APP}" - chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "/run/uwsgi/app/${APP}" - ;; - esac -} - - uWSGI_app_available() { # usage: uWSGI_app_available <myapp.ini> local CONF="$1" @@ -1378,7 +1353,7 @@ uWSGI_app_available() { uWSGI_install_app() { - # usage: uWSGI_install_app [<template option> ...] <myapp.ini> + # usage: uWSGI_install_app [<template option> ...] <myapp.ini> [{owner} [{group} [{chmod}]]] # # <template option>: see install_template @@ -1390,11 +1365,10 @@ uWSGI_install_app() { *) pos_args+=("$i");; esac done - uWSGI_prepare_app "${pos_args[1]}" mkdir -p "${uWSGI_APPS_AVAILABLE}" install_template "${template_opts[@]}" \ "${uWSGI_APPS_AVAILABLE}/${pos_args[1]}" \ - root root 644 + "${pos_args[2]:-root}" "${pos_args[3]:-root}" "${pos_args[4]:-644}" uWSGI_enable_app "${pos_args[1]}" uWSGI_restart "${pos_args[1]}" info_msg "uWSGI app: ${pos_args[1]} is installed" @@ -1468,7 +1442,6 @@ uWSGI_enable_app() { mkdir -p "${uWSGI_APPS_ENABLED}" rm -f "${uWSGI_APPS_ENABLED}/${CONF}" ln -s "${uWSGI_APPS_AVAILABLE}/${CONF}" "${uWSGI_APPS_ENABLED}/${CONF}" - chown "${uWSGI_USER}:${uWSGI_GROUP}" "${uWSGI_APPS_ENABLED}/${CONF}" info_msg "enabled uWSGI app: ${CONF}" ;; *) diff --git a/utils/lib_install.sh b/utils/lib_install.sh index 04bec5d9a..44734e165 100755 --- a/utils/lib_install.sh +++ b/utils/lib_install.sh @@ -126,7 +126,7 @@ install_log_searx_instance() { if in_container; then # SearXNG is listening on 127.0.0.1 and not available from outside container - # in containers the service is listening on 0.0.0.0 (see lxc-searx.env) + # in containers the service is listening on 0.0.0.0 (see lxc-searxng.env) echo -e "---- container setup" echo -e " ${_BBlack}HINT:${_creset} SearXNG only listen on loopback device" \ "${_BBlack}inside${_creset} the container." @@ -198,10 +198,11 @@ if in_container; then # hint: Linux containers do not have DNS entries, lets use IPs SEARXNG_URL="http://$(primary_ip)" fi +# shellcheck disable=SC2034 PUBLIC_URL="${SEARXNG_URL}" source_dot_config -# shellcheck source=utils/lxc-searx.env -source "${REPO_ROOT}/utils/lxc-searx.env" +# shellcheck source=utils/lxc-searxng.env +source "${REPO_ROOT}/utils/lxc-searxng.env" in_container && lxc_set_suite_env diff --git a/utils/lib_redis.sh b/utils/lib_redis.sh index 5eaa1770f..ba1435a86 100755 --- a/utils/lib_redis.sh +++ b/utils/lib_redis.sh @@ -42,6 +42,8 @@ REDIS_GIT_URL="https://github.com/redis/redis.git" REDIS_GIT_TAG="${REDIS_GIT_TAG:-6.2.6}" REDIS_USER="searxng-redis" +REDIS_GROUP="searxng-redis" + REDIS_HOME="/usr/local/${REDIS_USER}" REDIS_HOME_BIN="${REDIS_HOME}/.local/bin" REDIS_ENV="${REDIS_HOME}/.redis_env" @@ -113,7 +115,7 @@ redis.devpkg() { case ${DIST_ID} in ubuntu|debian) - pkg_install git build-essential + pkg_install git build-essential gawk ;; arch) pkg_install git base-devel @@ -139,15 +141,20 @@ redis.build() { rst_title "get redis sources" section redis.src "${CACHE}/redis" - if ! required_commands gcc nm make gawk; then - sudo -H "$0" redis.devpkg + if ! required_commands gcc nm make gawk ; then + info_msg "install development tools to get missing command(s) .." + if [[ -n ${SUDO_USER} ]]; then + sudo -H "$0" redis.devpkg + else + redis.devpkg + fi fi rst_title "compile redis sources" section pushd "${CACHE}/redis" &>/dev/null - if ask_yn "Do you run 'make distclean' first'?" Ny; then + if ask_yn "Do you run 'make distclean' first'?" Yn; then $(bash.cmd) -c "make distclean" 2>&1 | prefix_stdout fi @@ -158,7 +165,7 @@ redis.build() { popd &>/dev/null - tee_stderr 0.1 <<EOF | $(bash.cmd) 2>&1 | prefix_stdout + tee_stderr 0.1 <<EOF | $(bash.cmd) 2>&1 | prefix_stdout mkdir -p "$(redis._get_dist)" cd "${CACHE}/redis/src" cp ${REDIS_INSTALL_EXE[@]} "$(redis._get_dist)" @@ -233,7 +240,7 @@ useradd --shell /bin/bash --system \ --home-dir "${REDIS_HOME}" \ --comment 'user that runs a redis instance' "${REDIS_USER}" mkdir -p "${REDIS_HOME}" -chown -R "${REDIS_USER}:${REDIS_USER}" "${REDIS_HOME}" +chown -R "${REDIS_USER}:${REDIS_GROUP}" "${REDIS_HOME}" groups "${REDIS_USER}" EOF @@ -248,7 +255,7 @@ EOF redis.userdel() { sudo_or_exit drop_service_account "${REDIS_USER}" - groupdel "${REDIS_USER}" 2>&1 | prefix_stdout || true + groupdel "${REDIS_GROUP}" 2>&1 | prefix_stdout || true } redis.addgrp() { @@ -256,7 +263,7 @@ redis.addgrp() { # usage: redis.addgrp <user> [[ -z $1 ]] && die_caller 42 "missing argument <user>" - sudo -H gpasswd -a "$1" "${REDIS_USER}" + sudo -H gpasswd -a "$1" "${REDIS_GROUP}" } redis.rmgrp() { @@ -264,7 +271,7 @@ redis.rmgrp() { # usage: redis.rmgrp <user> [[ -z $1 ]] && die_caller 42 "missing argument <user>" - sudo -H gpasswd -d "$1" "${REDIS_USER}" + sudo -H gpasswd -d "$1" "${REDIS_GROUP}" } @@ -278,7 +285,7 @@ redis._install_bin() { ( set -e for redis_exe in "${REDIS_INSTALL_EXE[@]}"; do - install -v -o "${REDIS_USER}" -g "${REDIS_USER}" \ + install -v -o "${REDIS_USER}" -g "${REDIS_GROUP}" \ "${src}/${redis_exe}" "${REDIS_HOME_BIN}" done diff --git a/utils/lxc-searx.env b/utils/lxc-searxng.env index 13b15522c..86279d4b3 100644 --- a/utils/lxc-searx.env +++ b/utils/lxc-searxng.env @@ -4,24 +4,18 @@ # This file is a setup of a LXC suite. It is sourced from different context, do # not manipulate the environment directly, implement functions and manipulate -# environment only is subshells! +# environment only in subshells. -# ---------------------------------------------------------------------------- -# config -# ---------------------------------------------------------------------------- - -# shellcheck disable=SC2034 -LXC_SUITE_NAME="searx" lxc_set_suite_env() { + + export LXC_SUITE_NAME="searxng" + # name of https://images.linuxcontainers.org export LINUXCONTAINERS_ORG_NAME="${LINUXCONTAINERS_ORG_NAME:-images}" export LXC_HOST_PREFIX="${LXC_SUITE_NAME:-searx}" export LXC_SUITE=( - # to disable containers, comment out lines .. - # end of standard support see https://wiki.ubuntu.com/Releases - "$LINUXCONTAINERS_ORG_NAME:ubuntu/18.04" "ubu1804" # April 2023 "$LINUXCONTAINERS_ORG_NAME:ubuntu/20.04" "ubu2004" # April 2025 "$LINUXCONTAINERS_ORG_NAME:ubuntu/21.10" "ubu2110" # July 2027 @@ -30,49 +24,27 @@ lxc_set_suite_env() { # rolling releases see https://www.archlinux.org/releng/releases/ "$LINUXCONTAINERS_ORG_NAME:archlinux" "archlinux" - - # EOL 30 June 2024 - "$LINUXCONTAINERS_ORG_NAME:centos/7" "centos7" ) - - PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}" - if in_container; then - # container hostnames do not have a DNS entry: use primary IP! - PUBLIC_URL="http://$(primary_ip)/searx" - - # make GUEST's services public to the HOST - FILTRON_API="0.0.0.0:4005" - FILTRON_LISTEN="0.0.0.0:4004" - MORTY_LISTEN="0.0.0.0:3000" - - # export LXC specific environment - export PUBLIC_URL FILTRON_API FILTRON_LISTEN MORTY_LISTEN - fi } lxc_suite_install_info() { ( lxc_set_suite_env cat <<EOF -LXC suite: ${LXC_SUITE_NAME} --> ${PUBLIC_URL} - suite includes searx, morty & filtron -suite images: -$(echo " ${LOCAL_IMAGES[*]}" | $FMT) -suite containers: -$(echo " ${CONTAINERS[*]}" | $FMT) +LXC suite: ${LXC_SUITE_NAME} + Suite includes installation of SearXNG + images: ${LOCAL_IMAGES[*]} + containers: ${CONTAINERS[*]} EOF ) - } +} lxc_suite_install() { ( lxc_set_suite_env FORCE_TIMEOUT=0 export FORCE_TIMEOUT - "${LXC_REPO_ROOT}/utils/searx.sh" install all - "${LXC_REPO_ROOT}/utils/morty.sh" install all - "${LXC_REPO_ROOT}/utils/filtron.sh" install all - + "${LXC_REPO_ROOT}/utils/searxng.sh" install all rst_title "suite installation finished ($(hostname))" part lxc_suite_info echo @@ -88,10 +60,9 @@ lxc_suite_info() { else # IPv4: # shellcheck disable=SC2034,SC2031 - info_msg "(${ip%|*}) filtron: http://${ip#*|}:4004/ $PUBLIC_URL" - info_msg "(${ip%|*}) morty: http://${ip#*|}:3000/ $PUBLIC_URL_MORTY" info_msg "(${ip%|*}) docs-live: http://${ip#*|}:8080/" fi done + "${LXC_REPO_ROOT}/utils/searxng.sh" searxng.instance.env ) } diff --git a/utils/lxc.sh b/utils/lxc.sh index 9754b5d75..5d16ef60e 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -9,7 +9,7 @@ source_dot_config source "${REPO_ROOT}/utils/brand.env" # load environment of the LXC suite -LXC_ENV="${LXC_ENV:-${REPO_ROOT}/utils/lxc-searx.env}" +LXC_ENV="${LXC_ENV:-${REPO_ROOT}/utils/lxc-searxng.env}" source "$LXC_ENV" lxc_set_suite_env diff --git a/utils/morty.sh b/utils/morty.sh index 28b76947e..c1ed60631 100755 --- a/utils/morty.sh +++ b/utils/morty.sh @@ -96,7 +96,7 @@ EOF install_log_searx_instance if in_container; then - # in containers the service is listening on 0.0.0.0 (see lxc-searx.env) + # in containers the service is listening on 0.0.0.0 (see lxc-searxng.env) for ip in $(global_IPs) ; do if [[ $ip =~ .*:.* ]]; then echo " container URL (IPv6): http://[${ip#*|}]:3000/" diff --git a/utils/searxng.sh b/utils/searxng.sh new file mode 100755 index 000000000..11dd0d675 --- /dev/null +++ b/utils/searxng.sh @@ -0,0 +1,894 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-or-later +# shellcheck disable=SC2001 + +# Script options from the environment: +SEARXNG_UWSGI_USE_SOCKET="${SEARXNG_UWSGI_USE_SOCKET:-true}" + +# shellcheck source=utils/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +# shellcheck source=utils/lib_redis.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib_redis.sh" +# shellcheck source=utils/brand.env +source "${REPO_ROOT}/utils/brand.env" + +SERVICE_NAME="searxng" +SERVICE_USER="searxng" +SERVICE_HOME="/usr/local/searxng" +SERVICE_GROUP="searxng" + +SEARXNG_SRC="${SERVICE_HOME}/searxng-src" +# shellcheck disable=SC2034 +SEARXNG_STATIC="${SEARXNG_SRC}/searx/static" + +SEARXNG_PYENV="${SERVICE_HOME}/searx-pyenv" +SEARXNG_SETTINGS_PATH="/etc/searxng/settings.yml" +SEARXNG_UWSGI_APP="searxng.ini" + +SEARXNG_INTERNAL_HTTP="${SEARXNG_BIND_ADDRESS}:${SEARXNG_PORT}" +if [[ ${SEARXNG_UWSGI_USE_SOCKET} == true ]]; then + SEARXNG_UWSGI_SOCKET="${SERVICE_HOME}/run/socket" +else + SEARXNG_UWSGI_SOCKET= +fi + +# SEARXNG_URL: the public URL of the instance (https://example.org/searxng). The +# value is taken from environment ${SEARXNG_URL} in ./utils/brand.env. This +# variable is an empty string if server.base_url in the settings.yml is set to +# 'false'. + +SEARXNG_URL="${SEARXNG_URL:-http://$(uname -n)/searxng}" +SEARXNG_URL="${SEARXNG_URL%/}" # if exists, remove trailing slash +if in_container; then + # hint: Linux containers do not have DNS entries, lets use IPs + SEARXNG_URL="http://$(primary_ip)/searxng" +fi +SEARXNG_URL_PATH="$(echo "${SEARXNG_URL}" | sed -e 's,^.*://[^/]*\(/.*\),\1,g')" +[[ "${SEARXNG_URL_PATH}" == "${SEARXNG_URL}" ]] && SEARXNG_URL_PATH=/ + +# Apache settings + +APACHE_SEARXNG_SITE="searxng.conf" + +# nginx settings + +NGINX_SEARXNG_SITE="searxng.conf" + +# apt packages + +SEARXNG_PACKAGES_debian="\ +python3-dev python3-babel python3-venv +uwsgi uwsgi-plugin-python3 +git build-essential libxslt-dev zlib1g-dev libffi-dev libssl-dev" + +SEARXNG_BUILD_PACKAGES_debian="\ +firefox graphviz imagemagick texlive-xetex librsvg2-bin +texlive-latex-recommended texlive-extra-utils fonts-dejavu +latexmk shellcheck" + +# pacman packages + +SEARXNG_PACKAGES_arch="\ +python python-pip python-lxml python-babel +uwsgi uwsgi-plugin-python +git base-devel libxml2" + +SEARXNG_BUILD_PACKAGES_arch="\ +firefox graphviz imagemagick texlive-bin extra/librsvg +texlive-core texlive-latexextra ttf-dejavu shellcheck" + +# dnf packages + +SEARXNG_PACKAGES_fedora="\ +python python-pip python-lxml python-babel python3-devel +uwsgi uwsgi-plugin-python3 +git @development-tools libxml2 openssl" + +SEARXNG_BUILD_PACKAGES_fedora="\ +firefox graphviz graphviz-gd ImageMagick librsvg2-tools +texlive-xetex-bin texlive-collection-fontsrecommended +texlive-collection-latex dejavu-sans-fonts dejavu-serif-fonts +dejavu-sans-mono-fonts ShellCheck" + +case $DIST_ID-$DIST_VERS in + ubuntu-18.04) + SEARXNG_PACKAGES="${SEARXNG_PACKAGES_debian}" + SEARXNG_BUILD_PACKAGES="${SEARXNG_BUILD_PACKAGES_debian}" + APACHE_PACKAGES="$APACHE_PACKAGES libapache2-mod-proxy-uwsgi" + ;; + ubuntu-20.04) + # https://wiki.ubuntu.com/FocalFossa/ReleaseNotes#Python3_by_default + SEARXNG_PACKAGES="${SEARXNG_PACKAGES_debian} python-is-python3" + SEARXNG_BUILD_PACKAGES="${SEARXNG_BUILD_PACKAGES_debian}" + ;; + ubuntu-*|debian-*) + SEARXNG_PACKAGES="${SEARXNG_PACKAGES_debian}" + SEARXNG_BUILD_PACKAGES="${SEARXNG_BUILD_PACKAGES_debian}" + ;; + arch-*) + SEARXNG_PACKAGES="${SEARXNG_PACKAGES_arch}" + SEARXNG_BUILD_PACKAGES="${SEARXNG_BUILD_PACKAGES_arch}" + ;; + fedora-*) + SEARXNG_PACKAGES="${SEARXNG_PACKAGES_fedora}" + SEARXNG_BUILD_PACKAGES="${SEARXNG_BUILD_PACKAGES_fedora}" + ;; +esac + +_service_prefix=" ${_Yellow}|${SERVICE_USER}|${_creset} " + +# ---------------------------------------------------------------------------- +usage() { +# ---------------------------------------------------------------------------- + + # shellcheck disable=SC1117 + cat <<EOF +usage: + $(basename "$0") install [all|user|pyenv|settings|uwsgi|redis|nginx|apache|searxng-src|packages|buildhost] + $(basename "$0") remove [all|user|pyenv|settings|uwsgi|redis|nginx|apache] + $(basename "$0") instance [cmd|update|check|localtest|inspect] +install|remove: + all : complete (de-) installation of the SearXNG service + user : service user '${SERVICE_USER}' (${SERVICE_HOME}) + pyenv : virtualenv (python) in ${SEARXNG_PYENV} + settings : settings from ${SEARXNG_SETTINGS_PATH} + uwsgi : SearXNG's uWSGI app ${SEARXNG_UWSGI_APP} + redis : build & install or remove a local redis server ${REDIS_HOME}/run/redis.sock + nginx : HTTP site ${NGINX_APPS_AVAILABLE}/${NGINX_SEARXNG_SITE} + apache : HTTP site ${APACHE_SITES_AVAILABLE}/${APACHE_SEARXNG_SITE} +install: + searxng-src : clone ${GIT_URL} into ${SEARXNG_SRC} + packages : installs packages from OS package manager required by SearXNG + buildhost : installs packages from OS package manager required by a SearXNG buildhost +instance: + update : update SearXNG instance (git fetch + reset & update settings.yml) + check : run checks from utils/searxng_check.py in the active installation + inspect : run some small tests and inspect SearXNG's server status and log + get_setting : get settings value from running SearXNG instance + cmd : run command in SearXNG instance's environment (e.g. bash) +EOF + searxng.instance.env + [[ -n ${1} ]] && err_msg "$1" +} + +searxng.instance.env() { + echo "uWSGI:" + if [[ ${SEARXNG_UWSGI_USE_SOCKET} == true ]]; then + echo " SEARXNG_UWSGI_SOCKET : ${SEARXNG_UWSGI_SOCKET}" + else + echo " SEARXNG_INTERNAL_HTTP: ${SEARXNG_INTERNAL_HTTP}" + fi + cat <<EOF +environment ${SEARXNG_SRC}/utils/brand.env: + GIT_URL : ${GIT_URL} + GIT_BRANCH : ${GIT_BRANCH} + SEARXNG_URL : ${SEARXNG_URL} + SEARXNG_PORT : ${SEARXNG_PORT} + SEARXNG_BIND_ADDRESS : ${SEARXNG_BIND_ADDRESS} +EOF +} + +main() { + required_commands \ + sudo systemctl install git wget curl \ + || exit + + local _usage="unknown or missing $1 command $2" + + case $1 in + --getenv) var="$2"; echo "${!var}"; exit 0;; + -h|--help) usage; exit 0;; + install) + sudo_or_exit + case $2 in + all) searxng.install.all;; + user) searxng.install.user;; + pyenv) searxng.install.pyenv;; + searxng-src) searxng.install.clone;; + settings) searxng.install.settings;; + uwsgi) searxng.install.uwsgi;; + packages) searxng.install.packages;; + buildhost) searxng.install.buildhost;; + nginx) searxng.nginx.install;; + apache) searxng.apache.install;; + redis) searxng.install.redis;; + *) usage "$_usage"; exit 42;; + esac + ;; + remove) + sudo_or_exit + case $2 in + all) searxng.remove.all;; + user) drop_service_account "${SERVICE_USER}";; + pyenv) searxng.remove.pyenv;; + settings) searxng.remove.settings;; + uwsgi) searxng.remove.uwsgi;; + apache) searxng.apache.remove;; + remove) searxng.nginx.remove;; + redis) searxng.remove.redis;; + *) usage "$_usage"; exit 42;; + esac + ;; + instance) + case $2 in + update) + sudo_or_exit + searxng.instance.update + ;; + check) + sudo_or_exit + searxng.instance.self.call searxng.check + ;; + inspect) + sudo_or_exit + searxng.instance.inspect + ;; + cmd) + sudo_or_exit + shift; shift; searxng.instance.exec "$@" + ;; + get_setting) + shift; shift; searxng.instance.get_setting "$@" + ;; + call) + # call a function in instance's environment + shift; shift; searxng.instance.self.call "$@" + ;; + _call) + shift; shift; "$@" + ;; + *) usage "$_usage"; exit 42;; + esac + ;; + *) + local cmd="$1" + _type="$(type -t "$cmd")" + if [ "$_type" != 'function' ]; then + usage "unknown or missing command $1" + exit 42 + else + "$cmd" "$@" + fi + ;; + esac +} + +searxng.install.all() { + rst_title "SearXNG installation" part + + local redis_url + + rst_title "SearXNG" + searxng.install.packages + wait_key 10 + searxng.install.user + wait_key 10 + searxng.install.clone + wait_key + searxng.install.pyenv + wait_key + searxng.install.settings + wait_key + searxng.instance.localtest + wait_key + searxng.install.uwsgi + wait_key + + rst_title "Redis DB" + searxng.install.redis.db + + rst_title "HTTP Server" + searxng.install.http.site + + rst_title "Finalize installation" + if ask_yn "Do you want to run some checks?" Yn; then + searxng.instance.self.call searxng.check + fi +} + +searxng.install.redis.db() { + local redis_url + + redis_url=$(searxng.instance.get_setting redis.url) + rst_para "\ +In your instance, redis DB connector is configured at: + + ${redis_url} +" + if searxng.instance.exec python -c "from searx.shared import redisdb; redisdb.init() or exit(42)"; then + info_msg "SearXNG instance is able to connect redis DB." + return + fi + if ! [[ ${redis_url} = unix://${REDIS_HOME}/run/redis.sock* ]]; then + err_msg "SearXNG instance can't connect redis DB / check redis & your settings" + return + fi + rst_para ".. but this redis DB is not installed yet." + + case $DIST_ID-$DIST_VERS in + fedora-*) + # Fedora runs uWSGI in emperor-tyrant mode: in Tyrant mode the + # Emperor will run the vassal using the UID/GID of the vassal + # configuration file [1] (user and group of the app .ini file). + # + # HINT: without option ``emperor-tyrant-initgroups=true`` in + # ``/etc/uwsgi.ini`` the process won't get the additional groups, + # but this option is not available in 2.0.x branch [2][3] / on + # fedora35 there is v2.0.20 installed --> no way to get additional + # groups on fedora's tyrant mode. + # + # ERROR:searx.shared.redis: [searxng (993)] can't connect redis DB ... + # ERROR:searx.shared.redis: Error 13 connecting to unix socket: /usr/local/searxng-redis/run/redis.sock. Permission denied. + # ERROR:searx.plugins.limiter: init limiter DB failed!!! + # + # $ ps -aef | grep '/usr/sbin/uwsgi --ini searxng.ini' + # searxng 93 92 0 12:43 ? 00:00:00 /usr/sbin/uwsgi --ini searxng.ini + # searxng 186 93 0 12:44 ? 00:00:01 /usr/sbin/uwsgi --ini searxng.ini + # + # Additional groups: + # + # $ groups searxng + # searxng : searxng searxng-redis + # + # Here you can see that the additional "Groups" of PID 186 are unset + # (missing gid of searxng-redis) + # + # $ cat /proc/186/task/186/status + # ... + # Uid: 993 993 993 993 + # Gid: 993 993 993 993 + # FDSize: 128 + # Groups: + # ... + # + # [1] https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html#tyrant-mode-secure-multi-user-hosting + # [2] https://github.com/unbit/uwsgi/issues/2099 + # [3] https://github.com/unbit/uwsgi/pull/752 + + rst_para "\ +Fedora uses emperor-tyrant mode / in this mode we had a lot of trouble with +sockets and permissions of the vasals. We recommend to setup a redis DB +and using redis:// TCP protocol in the settings.yml configuration." + ;; + *) + if ask_yn "Do you want to install the redis DB now?" Yn; then + searxng.install.redis + uWSGI_restart "$SEARXNG_UWSGI_APP" + fi + ;; + esac +} + +searxng.install.http.site() { + + if apache_is_installed; then + info_msg "Apache is installed on this host." + if ask_yn "Do you want to install a reverse proxy" Yn; then + searxng.apache.install + fi + elif nginx_is_installed; then + info_msg "Nginx is installed on this host." + if ask_yn "Do you want to install a reverse proxy" Yn; then + searxng.nginx.install + fi + else + info_msg "Don't forget to install HTTP site." + fi +} + +searxng.remove.all() { + local redis_url + + rst_title "De-Install SearXNG (service)" + if ! ask_yn "Do you really want to deinstall SearXNG?"; then + return + fi + + redis_url=$(searxng.instance.get_setting redis.url) + if ! [[ ${redis_url} = unix://${REDIS_HOME}/run/redis.sock* ]]; then + searxng.remove.redis + fi + + searxng.remove.uwsgi + drop_service_account "${SERVICE_USER}" + searxng.remove.settings + wait_key + + if service_is_available "${SEARXNG_URL}"; then + MSG="** Don't forgett to remove your public site! (${SEARXNG_URL}) **" wait_key 10 + fi +} + +searxng.install.user() { + rst_title "SearXNG -- install user" section + echo + if getent passwd "${SERVICE_USER}" > /dev/null; then + echo "user already exists" + return 0 + fi + + tee_stderr 1 <<EOF | bash | prefix_stdout +useradd --shell /bin/bash --system \ + --home-dir "${SERVICE_HOME}" \ + --comment 'Privacy-respecting metasearch engine' ${SERVICE_USER} +mkdir "${SERVICE_HOME}" +chown -R "${SERVICE_GROUP}:${SERVICE_GROUP}" "${SERVICE_HOME}" +groups ${SERVICE_USER} +EOF +} + +searxng.install.packages() { + TITLE="SearXNG -- install packages" pkg_install "${SEARXNG_PACKAGES}" +} + +searxng.install.buildhost() { + TITLE="SearXNG -- install buildhost packages" pkg_install \ + "${SEARXNG_PACKAGES} ${SEARXNG_BUILD_PACKAGES}" +} + +searxng.install.clone() { + rst_title "Clone SearXNG sources" section + if ! service_account_is_available "${SERVICE_USER}"; then + die 42 "To clone SearXNG, first install user ${SERVICE_USER}." + fi + echo + if ! sudo -i -u "${SERVICE_USER}" ls -d "$REPO_ROOT" > /dev/null; then + die 42 "user '${SERVICE_USER}' missed read permission: $REPO_ROOT" + fi + # SERVICE_HOME="$(sudo -i -u "${SERVICE_USER}" echo \$HOME 2>/dev/null)" + if [[ ! "${SERVICE_HOME}" ]]; then + err_msg "to clone SearXNG sources, user ${SERVICE_USER} hast to be created first" + return 42 + fi + if [[ ! $(git show-ref "refs/heads/${GIT_BRANCH}") ]]; then + warn_msg "missing local branch ${GIT_BRANCH}" + info_msg "create local branch ${GIT_BRANCH} from start point: origin/${GIT_BRANCH}" + git branch "${GIT_BRANCH}" "origin/${GIT_BRANCH}" + fi + if [[ ! $(git rev-parse --abbrev-ref HEAD) == "${GIT_BRANCH}" ]]; then + warn_msg "take into account, installing branch $GIT_BRANCH while current branch is $(git rev-parse --abbrev-ref HEAD)" + fi + # export SERVICE_HOME + + # clone repo and add a safe.directory entry to git's system config / see + # https://github.com/searxng/searxng/issues/1251 + git_clone "$REPO_ROOT" "${SEARXNG_SRC}" \ + "$GIT_BRANCH" "${SERVICE_USER}" + git config --system --add safe.directory "${SEARXNG_SRC}" + + pushd "${SEARXNG_SRC}" > /dev/null + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +cd "${SEARXNG_SRC}" +git remote set-url origin ${GIT_URL} +git config user.email "${ADMIN_EMAIL}" +git config user.name "${ADMIN_NAME}" +git config --list +EOF + popd > /dev/null +} + +searxng.install.pyenv() { + rst_title "Create virtualenv (python)" section + echo + if [[ ! -f "${SEARXNG_SRC}/manage" ]]; then + die 42 "To create pyenv for SearXNG, first install searxng-src." + fi + info_msg "create pyenv in ${SEARXNG_PYENV}" + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +rm -rf "${SEARXNG_PYENV}" +python3 -m venv "${SEARXNG_PYENV}" +grep -qFs -- 'source ${SEARXNG_PYENV}/bin/activate' ~/.profile \ + || echo 'source ${SEARXNG_PYENV}/bin/activate' >> ~/.profile +EOF + info_msg "inspect python's virtual environment" + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +command -v python && python --version +EOF + wait_key + info_msg "install needed python packages" + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +pip install -U pip +pip install -U setuptools +pip install -U wheel +pip install -U pyyaml +cd ${SEARXNG_SRC} +pip install -e . +EOF +} + +searxng.remove.pyenv() { + rst_title "Remove virtualenv (python)" section + if ! ask_yn "Do you really want to drop ${SEARXNG_PYENV} ?"; then + return + fi + info_msg "remove pyenv activation from ~/.profile" + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +grep -v 'source ${SEARXNG_PYENV}/bin/activate' ~/.profile > ~/.profile.## +mv ~/.profile.## ~/.profile +EOF + rm -rf "${SEARXNG_PYENV}" +} + +searxng.install.settings() { + rst_title "install ${SEARXNG_SETTINGS_PATH}" section + + if ! [[ -f "${SEARXNG_SRC}/.git/config" ]]; then + die "Before install settings, first install SearXNG." + exit 42 + fi + + mkdir -p "$(dirname "${SEARXNG_SETTINGS_PATH}")" + + DEFAULT_SELECT=1 \ + install_template --no-eval \ + "${SEARXNG_SETTINGS_PATH}" \ + "${SERVICE_USER}" "${SERVICE_GROUP}" + + tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "root" +sed -i -e "s/ultrasecretkey/$(openssl rand -hex 16)/g" "${SEARXNG_SETTINGS_PATH}" +EOF +} + +searxng.remove.settings() { + rst_title "remove ${SEARXNG_SETTINGS_PATH}" section + if ask_yn "Do you want to delete the SearXNG settings?" Yn; then + rm -f "${SEARXNG_SETTINGS_PATH}" + fi +} + +searxng.check() { + rst_title "SearXNG checks" section + + for NAME in "searx" "filtron" "morty"; do + if service_account_is_available "${NAME}"; then + err_msg "There exists an old '${NAME}' account from a previous installation." + else + info_msg "[OK] (old) account '${NAME}' does not exists" + fi + done + + "${SEARXNG_PYENV}/bin/python" "${SEARXNG_SRC}/utils/searxng_check.py" +} + +searxng.instance.update() { + rst_title "Update SearXNG instance" + rst_para "fetch from $GIT_URL and reset to origin/$GIT_BRANCH" + tee_stderr 0.3 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARXNG_SRC} +git fetch origin "$GIT_BRANCH" +git reset --hard "origin/$GIT_BRANCH" +pip install -U pip +pip install -U setuptools +pip install -U wheel +pip install -U pyyaml +pip install -U -e . +EOF + rst_para "update instance's settings.yml from ${SEARXNG_SETTINGS_PATH}" + DEFAULT_SELECT=2 \ + install_template --no-eval \ + "${SEARXNG_SETTINGS_PATH}" \ + "${SERVICE_USER}" "${SERVICE_GROUP}" + + sudo -H -i <<EOF +sed -i -e "s/ultrasecretkey/$(openssl rand -hex 16)/g" "${SEARXNG_SETTINGS_PATH}" +EOF + uWSGI_restart "${SEARXNG_UWSGI_APP}" +} + +searxng.install.uwsgi() { + rst_title "SearXNG (install uwsgi)" + install_uwsgi + if [[ ${SEARXNG_UWSGI_USE_SOCKET} == true ]]; then + searxng.install.uwsgi.socket + else + searxng.install.uwsgi.http + fi +} + +searxng.install.uwsgi.http() { + rst_para "Install ${SEARXNG_UWSGI_APP} at: http://${SEARXNG_INTERNAL_HTTP}" + uWSGI_install_app "${SEARXNG_UWSGI_APP}" + if ! searxng.uwsgi.available; then + err_msg "URL http://${SEARXNG_INTERNAL_HTTP} not available, check SearXNG & uwsgi setup!" + fi +} + +searxng.install.uwsgi.socket() { + rst_para "Install ${SEARXNG_UWSGI_APP} using socket at: ${SEARXNG_UWSGI_SOCKET}" + mkdir -p "$(dirname ${SEARXNG_UWSGI_SOCKET})" + chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "$(dirname ${SEARXNG_UWSGI_SOCKET})" + + case $DIST_ID-$DIST_VERS in + fedora-*) + # Fedora runs uWSGI in emperor-tyrant mode: in Tyrant mode the + # Emperor will run the vassal using the UID/GID of the vassal + # configuration file [1] (user and group of the app .ini file). + # [1] https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html#tyrant-mode-secure-multi-user-hosting + uWSGI_install_app --variant=socket "${SEARXNG_UWSGI_APP}" "${SERVICE_USER}" "${SERVICE_GROUP}" + ;; + *) + uWSGI_install_app --variant=socket "${SEARXNG_UWSGI_APP}" + ;; + esac + sleep 5 + if ! searxng.uwsgi.available; then + err_msg "uWSGI socket not available at: ${SEARXNG_UWSGI_SOCKET}" + fi +} + +searxng.uwsgi.available() { + if [[ ${SEARXNG_UWSGI_USE_SOCKET} == true ]]; then + [[ -S "${SEARXNG_UWSGI_SOCKET}" ]] + exit_val=$? + if [[ $exit_val = 0 ]]; then + info_msg "uWSGI socket is located at: ${SEARXNG_UWSGI_SOCKET}" + fi + else + service_is_available "http://${SEARXNG_INTERNAL_HTTP}" + exit_val=$? + fi + return "$exit_val" +} + +searxng.remove.uwsgi() { + rst_title "Remove SearXNG's uWSGI app (${SEARXNG_UWSGI_APP})" section + echo + uWSGI_remove_app "${SEARXNG_UWSGI_APP}" +} + +searxng.install.redis() { + rst_title "SearXNG (install redis)" + redis.build + redis.install + redis.addgrp "${SERVICE_USER}" +} + +searxng.remove.redis() { + rst_title "SearXNG (remove redis)" + redis.rmgrp "${SERVICE_USER}" + redis.remove +} + +searxng.instance.localtest() { + rst_title "Test SearXNG instance localy" section + rst_para "Activate debug mode, start a minimal SearXNG "\ + "service and debug a HTTP request/response cycle." + + if service_is_available "http://${SEARXNG_INTERNAL_HTTP}" &>/dev/null; then + err_msg "URL/port http://${SEARXNG_INTERNAL_HTTP} is already in use, you" + err_msg "should stop that service before starting local tests!" + if ! ask_yn "Continue with local tests?"; then + return + fi + fi + echo + searxng.instance.debug.on + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +export SEARXNG_SETTINGS_PATH="${SEARXNG_SETTINGS_PATH}" +cd ${SEARXNG_SRC} +timeout 10 python searx/webapp.py & +sleep 3 +curl --location --verbose --head --insecure ${SEARXNG_INTERNAL_HTTP} +EOF + echo + searxng.instance.debug.off +} + +searxng.install.http.pre() { + if ! searxng.uwsgi.available; then + rst_para "\ +To install uWSGI use:: + + $(basename "$0") install uwsgi +" + die 42 "SearXNG's uWSGI app not available" + fi + + if ! searxng.instance.exec python -c "from searx.shared import redisdb; redisdb.init() or exit(42)"; then + rst_para "\ +The configured redis DB is not available: If your server is public to the +internet, you should setup a bot protection to block excessively bot queries. +Bot protection requires a redis DB. About bot protection visit the official +SearXNG documentation and query for the word 'limiter'. +" + fi +} + +searxng.apache.install() { + rst_title "Install Apache site ${APACHE_SEARXNG_SITE}" + rst_para "\ +This installs SearXNG's uWSGI app as apache site. The apache site is located at: +${APACHE_SITES_AVAILABLE}/${APACHE_SEARXNG_SITE}." + searxng.install.http.pre + + if ! apache_is_installed; then + err_msg "Apache packages are not installed" + if ! ask_yn "Do you really want to continue and install apache packages?" Yn; then + return + else + FORCE_SELECTION=Y install_apache + fi + else + info_msg "Apache packages are installed [OK]" + fi + + if [[ ${SEARXNG_UWSGI_USE_SOCKET} == true ]]; then + apache_install_site --variant=socket "${APACHE_SEARXNG_SITE}" + else + apache_install_site "${APACHE_SEARXNG_SITE}" + fi + + if ! service_is_available "${SEARXNG_URL}"; then + err_msg "Public service at ${SEARXNG_URL} is not available!" + fi +} + +searxng.apache.remove() { + rst_title "Remove Apache site ${APACHE_SEARXNG_SITE}" + rst_para "\ +This removes apache site ${APACHE_SEARXNG_SITE}:: + + ${APACHE_SITES_AVAILABLE}/${APACHE_SEARXNG_SITE}" + + ! apache_is_installed && err_msg "Apache is not installed." + if ! ask_yn "Do you really want to continue?" Yn; then + return + fi + apache_remove_site "${APACHE_SEARXNG_SITE}" +} + +searxng.nginx.install() { + + rst_title "Install nginx site ${NGINX_SEARXNG_SITE}" + rst_para "\ +This installs SearXNG's uWSGI app as Nginx site. The Nginx site is located at: +${NGINX_APPS_AVAILABLE}/${NGINX_SEARXNG_SITE} and requires a uWSGI." + searxng.install.http.pre + + if ! nginx_is_installed ; then + err_msg "Nginx packages are not installed" + if ! ask_yn "Do you really want to continue and install Nginx packages?" Yn; then + return + else + FORCE_SELECTION=Y install_nginx + fi + else + info_msg "Nginx packages are installed [OK]" + fi + + if [[ ${SEARXNG_UWSGI_USE_SOCKET} == true ]]; then + nginx_install_app --variant=socket "${NGINX_SEARXNG_SITE}" + else + nginx_install_app "${NGINX_SEARXNG_SITE}" + fi + + if ! service_is_available "${SEARXNG_URL}"; then + err_msg "Public service at ${SEARXNG_URL} is not available!" + fi +} + +searxng.nginx.remove() { + rst_title "Remove Nginx site ${NGINX_SEARXNG_SITE}" + rst_para "\ +This removes Nginx site ${NGINX_SEARXNG_SITE}:: + + ${NGINX_APPS_AVAILABLE}/${NGINX_SEARXNG_SITE}" + + ! nginx_is_installed && err_msg "Nginx is not installed." + if ! ask_yn "Do you really want to continue?" Yn; then + return + fi + nginx_remove_app "${NGINX_SEARXNG_SITE}" +} + +searxng.instance.exec() { + if ! service_account_is_available "${SERVICE_USER}"; then + die 42 "can't execute: instance does not exists (missed account ${SERVICE_USER})" + fi + sudo -H -i -u "${SERVICE_USER}" \ + SEARXNG_UWSGI_USE_SOCKET="${SEARXNG_UWSGI_USE_SOCKET}" \ + "$@" +} + +searxng.instance.self.call() { + # wrapper to call a function in instance's environment + info_msg "wrapper: utils/searxng.sh instance _call $*" + searxng.instance.exec "${SEARXNG_SRC}/utils/searxng.sh" instance _call "$@" +} + +searxng.instance.get_setting() { + searxng.instance.exec python <<EOF +from searx import get_setting +print(get_setting('$1')) +EOF +} + +searxng.instance.debug.on() { + warn_msg "Do not enable debug in a production environment!" + info_msg "try to enable debug mode ..." + tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARXNG_SRC} +sed -i -e "s/debug: false/debug: true/g" "$SEARXNG_SETTINGS_PATH" +EOF + uWSGI_restart "$SEARXNG_UWSGI_APP" +} + +searxng.instance.debug.off() { + info_msg "try to disable debug mode ..." + tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARXNG_SRC} +sed -i -e "s/debug: true/debug: false/g" "$SEARXNG_SETTINGS_PATH" +EOF + uWSGI_restart "$SEARXNG_UWSGI_APP" +} + +searxng.instance.inspect() { + rst_title "Inspect SearXNG instance" + echo + + searxng.instance.self.call _searxng.instance.inspect + + local _debug_on + if ask_yn "Enable SearXNG debug mode?"; then + searxng.instance.debug.on + _debug_on=1 + fi + echo + + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) + # For uWSGI debian uses the LSB init process; for each configuration + # file new uWSGI daemon instance is started with additional option. + service uwsgi status "${SERVICE_NAME}" + ;; + arch-*) + systemctl --no-pager -l status "uwsgi@${SERVICE_NAME%.*}" + ;; + fedora-*) + systemctl --no-pager -l status uwsgi + ;; + esac + + echo -e "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log" + read -r -s -n1 -t 5 + echo + + while true; do + trap break 2 + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) tail -f "/var/log/uwsgi/app/${SERVICE_NAME%.*}.log" ;; + arch-*) journalctl -f -u "uwsgi@${SERVICE_NAME%.*}" ;; + fedora-*) journalctl -f -u uwsgi ;; + esac + done + + if [[ $_debug_on == 1 ]]; then + searxng.instance.debug.off + fi + return 0 +} + +_searxng.instance.inspect() { + searxng.instance.env + + if in_container; then + # shellcheck source=utils/lxc-searxng.env + source "${REPO_ROOT}/utils/lxc-searxng.env" + lxc_suite_info + fi + + MSG="${_Green}[${_BCyan}CTRL-C${_Green}] to stop or [${_BCyan}KEY${_Green}] to continue${_creset}" + + if ! searxng.uwsgi.available; then + err_msg "SearXNG's uWSGI app not available" + wait_key + fi + if ! service_is_available "${SEARXNG_URL}"; then + err_msg "Public service at ${SEARXNG_URL} is not available!" + wait_key + fi +} + +# ---------------------------------------------------------------------------- +main "$@" +# ---------------------------------------------------------------------------- diff --git a/utils/searxng_check.py b/utils/searxng_check.py index afd0c4056..39e774340 100644 --- a/utils/searxng_check.py +++ b/utils/searxng_check.py @@ -25,3 +25,10 @@ if os.path.isfile(OLD_SETTING): os.environ.get('SEARXNG_SETTINGS_PATH', '/etc/searxng/settings.yml') )) warnings.warn(msg, DeprecationWarning) + +from searx.shared import redisdb +from searx import get_setting + +if not redisdb.init(): + warnings.warn("can't connect to redis DB at: %s" % get_setting('redis.url'), RuntimeWarning, stacklevel=2) + warnings.warn("--> no bot protection without redis DB", RuntimeWarning, stacklevel=2) diff --git a/utils/templates/etc/httpd/sites-available/searxng.conf b/utils/templates/etc/httpd/sites-available/searxng.conf new file mode 100644 index 000000000..5278640c3 --- /dev/null +++ b/utils/templates/etc/httpd/sites-available/searxng.conf @@ -0,0 +1,41 @@ +# -*- coding: utf-8; mode: apache -*- + +LoadModule ssl_module ${APACHE_MODULES}/mod_ssl.so +LoadModule headers_module ${APACHE_MODULES}/mod_headers.so +LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so +LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so +# LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so +# +# SetEnvIf Request_URI "${SEARXNG_URL_PATH}" dontlog +# CustomLog /dev/null combined env=dontlog + +<Location ${SEARXNG_URL_PATH}> + + Require all granted + Order deny,allow + Deny from all + # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + # add the trailing slash + RedirectMatch 308 ${SEARXNG_URL_PATH}\$ ${SEARXNG_URL_PATH}/ + + ProxyPreserveHost On + ProxyPass http://${SEARXNG_INTERNAL_HTTP} + + # see flaskfix.py + RequestHeader set X-Scheme %{REQUEST_SCHEME}s + RequestHeader set X-Script-Name ${SEARXNG_URL_PATH} + + # see limiter.py + RequestHeader set X-Real-IP %{REMOTE_ADDR}s + RequestHeader append X-Forwarded-For %{REMOTE_ADDR}s + +</Location> + +# uWSGI serves the static files and in settings.yml we use:: +# +# ui: +# static_use_hash: true +# +# Alias ${SEARXNG_URL_PATH}/static/ ${SEARXNG_STATIC}/ diff --git a/utils/templates/etc/httpd/sites-available/searxng.conf:socket b/utils/templates/etc/httpd/sites-available/searxng.conf:socket new file mode 100644 index 000000000..b55ea7560 --- /dev/null +++ b/utils/templates/etc/httpd/sites-available/searxng.conf:socket @@ -0,0 +1,41 @@ +# -*- coding: utf-8; mode: apache -*- + +LoadModule ssl_module ${APACHE_MODULES}/mod_ssl.so +LoadModule headers_module ${APACHE_MODULES}/mod_headers.so +LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so +LoadModule proxy_uwsgi_module ${APACHE_MODULES}/mod_proxy_uwsgi.so +# LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so +# +# SetEnvIf Request_URI "${SEARXNG_URL_PATH}" dontlog +# CustomLog /dev/null combined env=dontlog + +<Location ${SEARXNG_URL_PATH}> + + Require all granted + Order deny,allow + Deny from all + # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + # add the trailing slash + RedirectMatch 308 ${SEARXNG_URL_PATH}\$ ${SEARXNG_URL_PATH}/ + + ProxyPreserveHost On + ProxyPass unix:${SEARXNG_UWSGI_SOCKET}|uwsgi://uwsgi-uds-searxng/ + + # see flaskfix.py + RequestHeader set X-Scheme %{REQUEST_SCHEME}s + RequestHeader set X-Script-Name ${SEARXNG_URL_PATH} + + # see limiter.py + RequestHeader set X-Real-IP %{REMOTE_ADDR}s + RequestHeader append X-Forwarded-For %{REMOTE_ADDR}s + +</Location> + +# uWSGI serves the static files and in settings.yml we use:: +# +# ui: +# static_use_hash: true +# +# Alias ${SEARXNG_URL_PATH}/static/ ${SEARXNG_STATIC}/ diff --git a/utils/templates/etc/nginx/default.apps-available/searxng.conf b/utils/templates/etc/nginx/default.apps-available/searxng.conf new file mode 100644 index 000000000..7225a8f96 --- /dev/null +++ b/utils/templates/etc/nginx/default.apps-available/searxng.conf @@ -0,0 +1,29 @@ +location ${SEARXNG_URL_PATH} { + + proxy_pass http://${SEARXNG_INTERNAL_HTTP}; + + proxy_set_header Host \$host; + proxy_set_header Connection \$http_connection; + + # see flaskfix.py + proxy_set_header X-Scheme \$scheme; + proxy_set_header X-Script-Name ${SEARXNG_URL_PATH}; + + # see limiter.py + proxy_set_header X-Real-IP \$remote_addr; + proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; + + # proxy_buffering off; + # proxy_request_buffering off; + # proxy_buffer_size 8k; + +} + +# uWSGI serves the static files and in settings.yml we use:: +# +# ui: +# static_use_hash: true +# +# location ${SEARXNG_URL_PATH}/static/ { +# alias ${SEARXNG_STATIC}/; +# } diff --git a/utils/templates/etc/nginx/default.apps-available/searxng.conf:socket b/utils/templates/etc/nginx/default.apps-available/searxng.conf:socket new file mode 100644 index 000000000..7a74eab48 --- /dev/null +++ b/utils/templates/etc/nginx/default.apps-available/searxng.conf:socket @@ -0,0 +1,26 @@ +location ${SEARXNG_URL_PATH} { + + uwsgi_pass unix://${SEARXNG_UWSGI_SOCKET}; + + include uwsgi_params; + + uwsgi_param HTTP_HOST \$host; + uwsgi_param HTTP_CONNECTION \$http_connection; + + # see flaskfix.py + uwsgi_param HTTP_X_SCHEME \$scheme; + uwsgi_param HTTP_X_SCRIPT_NAME ${SEARXNG_URL_PATH}; + + # see limiter.py + uwsgi_param HTTP_X_REAL_IP \$remote_addr; + uwsgi_param HTTP_X_FORWARDED_FOR \$proxy_add_x_forwarded_for; +} + +# uWSGI serves the static files and in settings.yml we use:: +# +# ui: +# static_use_hash: true +# +# location ${SEARXNG_URL_PATH}/static/ { +# alias ${SEARXNG_STATIC}/; +# } diff --git a/utils/templates/etc/searxng/settings.yml b/utils/templates/etc/searxng/settings.yml index 860f4f5e9..aee21474d 100644 --- a/utils/templates/etc/searxng/settings.yml +++ b/utils/templates/etc/searxng/settings.yml @@ -1,46 +1,55 @@ -# SearXNG settings, before editing this file read: -# -# https://docs.searxng.org/admin/engines/settings.html +# SearXNG settings use_default_settings: true general: - # Debug mode, only for development debug: false - # change displayed name - # instance_name: "SearXNG" + instance_name: "SearXNG" search: - # Filter results. 0: None, 1: Moderate, 2: Strict - safe_search: 0 - # Existing autocomplete backends: "dbpedia", "duckduckgo", "google", - # "startpage", "swisscows", "qwant", "wikipedia" - leave blank to turn it off - # by default. - autocomplete: '' - # Default search language - leave blank to detect from browser information or - # use codes from 'languages.py' - default_lang: '' - # remove format to deny access, use lower case. - formats: - - html + safe_search: 2 + autocomplete: 'duckduckgo' server: - secret_key: "ultrasecretkey" # change this! - # Proxying image results through SearXNG - image_proxy: false + secret_key: "ultrasecretkey" + limiter: true + image_proxy: true + +redis: + url: unix:///usr/local/searxng-redis/run/redis.sock?db=0 + +ui: + static_use_hash: true -# result_proxy: -# url: http://127.0.0.1:3000/ -# key: !!binary "your_morty_proxy_key" +# preferences: +# lock: +# - autocomplete +# - method + +enabled_plugins: + - 'Hash plugin' + - 'Search on category select' + - 'Self Informations' + - 'Tracker URL remover' + - 'Ahmia blacklist' + # - 'Hostname replace' # see hostname_replace configuration below + # - 'Infinite scroll' + # - 'Open Access DOI rewrite' + # - 'Vim-like hotkeys' # plugins: # - only_show_green_results -# engines: -# -# - name: duckduckgo -# disabled: false +# hostname_replace: # +# # twitter --> nitter +# '(www\.)?twitter\.com$': 'nitter.net' + +engines: + + - name: google + use_mobile_ui: true + # - name: fdroid # disabled: false # @@ -48,6 +57,13 @@ server: # disabled: false # # - name: mediathekviewweb -# engine: mediathekviewweb -# shortcut: mvw -# categories: general +# categories: TV +# disabled: false +# +# - name: invidious +# disabled: false +# base_url: +# - https://invidious.snopyta.org +# - https://invidious.tiekoetter.com +# - https://invidio.xamh.de +# - https://inv.riverside.rocks diff --git a/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini b/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini index 931746306..04c32c662 100644 --- a/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini +++ b/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini @@ -65,29 +65,19 @@ pythonpath = ${SEARXNG_SRC} # speak to upstream # ----------------- -# -# Activate the 'http' configuration for filtron or activate the 'socket' -# configuration if you setup your HTTP server to use uWSGI protocol via sockets. -# using IP: -# # https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http # Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html http = ${SEARXNG_INTERNAL_HTTP} -# using unix-sockets: -# -# Don't forget to create the folder where the sockets should take place:: +# uWSGI serves the static files and in settings.yml we use:: # -# mkdir -p "$(dirname ${SEARXNG_UWSGI_SOCKET})" -# chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "$(dirname ${SEARXNG_UWSGI_SOCKET})" +# ui: +# static_use_hash: true # -# socket = ${SEARXNG_UWSGI_SOCKET} - -# uwsgi serves the static files -# expires set to one year since there are hashes static-map = /static=${SEARXNG_STATIC} +# expires set to one year since there are hashes static-expires = /* 31557600 static-gzip-all = True offload-threads = %k diff --git a/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini:socket b/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini:socket index 158973140..bbfaf63be 100644 --- a/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini:socket +++ b/utils/templates/etc/uwsgi/apps-archlinux/searxng.ini:socket @@ -65,24 +65,7 @@ pythonpath = ${SEARXNG_SRC} # speak to upstream # ----------------- -# -# Activate the 'http' configuration for filtron or activate the 'socket' -# configuration if you setup your HTTP server to use uWSGI protocol via sockets. - -# using IP: -# -# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http -# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html - -# http = ${SEARXNG_INTERNAL_HTTP} -# using unix-sockets: -# -# Don't forget to create the folder where the sockets should take place:: -# -# mkdir -p "$(dirname ${SEARXNG_UWSGI_SOCKET})" -# chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "$(dirname ${SEARXNG_UWSGI_SOCKET})" -# socket = ${SEARXNG_UWSGI_SOCKET} # uWSGI serves the static files and in settings.yml we use:: diff --git a/utils/templates/etc/uwsgi/apps-available/searxng.ini b/utils/templates/etc/uwsgi/apps-available/searxng.ini index 1926b446d..5ea7d991a 100644 --- a/utils/templates/etc/uwsgi/apps-available/searxng.ini +++ b/utils/templates/etc/uwsgi/apps-available/searxng.ini @@ -6,7 +6,11 @@ # # https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core -# Who will run the code +# Who will run the code / Hint: in emperor-tyrant mode uid & gid setting will be +# ignored [1]. Mode emperor-tyrant is the default on fedora (/etc/uwsgi.ini). +# +# [1] https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html#tyrant-mode-secure-multi-user-hosting +# uid = ${SERVICE_USER} gid = ${SERVICE_GROUP} @@ -64,29 +68,19 @@ pythonpath = ${SEARXNG_SRC} # speak to upstream # ----------------- -# -# Activate the 'http' configuration for filtron or activate the 'socket' -# configuration if you setup your HTTP server to use uWSGI protocol via sockets. -# using IP: -# # https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http # Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html http = ${SEARXNG_INTERNAL_HTTP} -# using unix-sockets: +# uWSGI serves the static files and in settings.yml we use:: # -# Don't forget to create the folder where the sockets should take place:: +# ui: +# static_use_hash: true # -# mkdir -p "$(dirname ${SEARXNG_UWSGI_SOCKET})" -# chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "$(dirname ${SEARXNG_UWSGI_SOCKET})" -# -# socket = ${SEARXNG_UWSGI_SOCKET} - -# uwsgi serves the static files -# expires set to one year since there are hashes static-map = /static=${SEARXNG_STATIC} +# expires set to one year since there are hashes static-expires = /* 31557600 static-gzip-all = True offload-threads = %k diff --git a/utils/templates/etc/uwsgi/apps-available/searxng.ini:socket b/utils/templates/etc/uwsgi/apps-available/searxng.ini:socket index 796d18b8e..304ea3500 100644 --- a/utils/templates/etc/uwsgi/apps-available/searxng.ini:socket +++ b/utils/templates/etc/uwsgi/apps-available/searxng.ini:socket @@ -6,7 +6,11 @@ # # https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core -# Who will run the code +# Who will run the code / Hint: in emperor-tyrant mode uid & gid setting will be +# ignored [1]. Mode emperor-tyrant is the default on fedora (/etc/uwsgi.ini). +# +# [1] https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html#tyrant-mode-secure-multi-user-hosting +# uid = ${SERVICE_USER} gid = ${SERVICE_GROUP} @@ -64,24 +68,7 @@ pythonpath = ${SEARXNG_SRC} # speak to upstream # ----------------- -# -# Activate the 'http' configuration for filtron or activate the 'socket' -# configuration if you setup your HTTP server to use uWSGI protocol via sockets. - -# using IP: -# -# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http -# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html -# http = ${SEARXNG_INTERNAL_HTTP} - -# using unix-sockets: -# -# Don't forget to create the folder where the sockets should take place:: -# -# mkdir -p "$(dirname ${SEARXNG_UWSGI_SOCKET})" -# chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "$(dirname ${SEARXNG_UWSGI_SOCKET})" -# socket = ${SEARXNG_UWSGI_SOCKET} # uWSGI serves the static files and in settings.yml we use:: |