diff options
| author | Markus Heiser <markus.heiser@darmarIT.de> | 2020-06-19 04:51:48 +0000 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2020-06-19 04:51:48 +0000 |
| commit | 4eab5cf854cdadea07f34d22a801c05e73730c5c (patch) | |
| tree | 980b13e958ee73fba1ef8e559deba8680c26085f /utils | |
| parent | 223430ff30549631be17ea36ff6d1f119c6af64e (diff) | |
| parent | c83007a6bcd2a7f765e7655b91cc6714fbc4ab01 (diff) | |
| download | searxng-4eab5cf854cdadea07f34d22a801c05e73730c5c.tar.gz searxng-4eab5cf854cdadea07f34d22a801c05e73730c5c.zip | |
Merge branch 'master' into gigablast
Diffstat (limited to 'utils')
25 files changed, 4875 insertions, 16 deletions
diff --git a/utils/brand.env b/utils/brand.env index 7fe1a3911..5d5b39513 100644 --- a/utils/brand.env +++ b/utils/brand.env @@ -1,4 +1,5 @@ export GIT_URL='https://github.com/asciimoo/searx' +export GIT_BRANCH='master' export ISSUE_URL='https://github.com/asciimoo/searx/issues' export SEARX_URL='https://searx.me' export DOCS_URL='https://asciimoo.github.io/searx' diff --git a/utils/filtron.sh b/utils/filtron.sh new file mode 100755 index 000000000..8986fb0ef --- /dev/null +++ b/utils/filtron.sh @@ -0,0 +1,561 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- +# SPDX-License-Identifier: AGPL-3.0-or-later +# shellcheck disable=SC2119,SC2001 + +# shellcheck source=utils/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +# shellcheck source=utils/brand.env +source "${REPO_ROOT}/utils/brand.env" +source_dot_config +source "${REPO_ROOT}/utils/lxc-searx.env" +in_container && lxc_set_suite_env + +# ---------------------------------------------------------------------------- +# config +# ---------------------------------------------------------------------------- + +PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}" +PUBLIC_HOST="${PUBLIC_HOST:-$(echo "$PUBLIC_URL" | sed -e 's/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/')}" + +FILTRON_URL_PATH="${FILTRON_URL_PATH:-$(echo "${PUBLIC_URL}" \ +| sed -e 's,^.*://[^/]*\(/.*\),\1,g')}" +[[ "${FILTRON_URL_PATH}" == "${PUBLIC_URL}" ]] && FILTRON_URL_PATH=/ + +FILTRON_ETC="/etc/filtron" +FILTRON_RULES="$FILTRON_ETC/rules.json" + +FILTRON_API="${FILTRON_API:-127.0.0.1:4005}" +FILTRON_LISTEN="${FILTRON_LISTEN:-127.0.0.1:4004}" +FILTRON_TARGET="${FILTRON_TARGET:-127.0.0.1:8888}" + +SERVICE_NAME="filtron" +SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}" +SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}" +SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}" +SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service" +# shellcheck disable=SC2034 +SERVICE_GROUP="${SERVICE_USER}" + +# shellcheck disable=SC2034 +SERVICE_GROUP="${SERVICE_USER}" + +GO_ENV="${SERVICE_HOME}/.go_env" +GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz" +GO_TAR=$(basename "$GO_PKG_URL") + +APACHE_FILTRON_SITE="searx.conf" +NGINX_FILTRON_SITE="searx.conf" + +# shellcheck disable=SC2034 +CONFIG_FILES=( + "${FILTRON_RULES}" + "${SERVICE_SYSTEMD_UNIT}" +) + +# ---------------------------------------------------------------------------- +usage() { +# ---------------------------------------------------------------------------- + + # shellcheck disable=SC1117 + cat <<EOF +usage:: + $(basename "$0") shell + $(basename "$0") install [all|user|rules] + $(basename "$0") update [filtron] + $(basename "$0") remove [all] + $(basename "$0") activate [service] + $(basename "$0") deactivate [service] + $(basename "$0") inspect [service] + $(basename "$0") option [debug-on|debug-off] + $(basename "$0") apache [install|remove] + $(basename "$0") nginx [install|remove] + +shell + start interactive shell from user ${SERVICE_USER} +install / remove + :all: complete setup of filtron service + :user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME) + :rules: reinstall filtron rules $FILTRON_RULES +update filtron + Update filtron installation ($SERVICE_HOME) +activate service + activate and start service daemon (systemd unit) +deactivate service + stop and deactivate service daemon (systemd unit) +inspect service + show service status and log +option + set one of the available options +apache (${PUBLIC_URL}) + :install: apache site with a reverse proxy (ProxyPass) + :remove: apache site ${APACHE_FILTRON_SITE} +nginx (${PUBLIC_URL}) + :install: nginx site with a reverse proxy (ProxyPass) + :remove: nginx site ${NGINX_FILTRON_SITE} + +filtron rules: ${FILTRON_RULES} + +If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/"}' file:: + PUBLIC_URL : ${PUBLIC_URL} + PUBLIC_HOST : ${PUBLIC_HOST} + SERVICE_USER : ${SERVICE_USER} + FILTRON_TARGET : ${FILTRON_TARGET} + FILTRON_API : ${FILTRON_API} + FILTRON_LISTEN : ${FILTRON_LISTEN} +EOF + if in_container; then + # in containers the service is listening on 0.0.0.0 (see lxc-searx.env) + for ip in $(global_IPs) ; do + if [[ $ip =~ .*:.* ]]; then + echo " container URL (IPv6): http://[${ip#*|}]:4005/" + else + # IPv4: + echo " container URL (IPv4): http://${ip#*|}:4005/" + fi + done + fi + [[ -n ${1} ]] && err_msg "$1" +} + +main() { + required_commands \ + sudo install git wget curl \ + || exit + + local _usage="unknown or missing $1 command $2" + + case $1 in + --getenv) var="$2"; echo "${!var}"; exit 0;; + -h|--help) usage; exit 0;; + + shell) + sudo_or_exit + interactive_shell "${SERVICE_USER}" + ;; + inspect) + case $2 in + service) + sudo_or_exit + inspect_service + ;; + *) usage "$_usage"; exit 42;; + esac ;; + install) + rst_title "$SERVICE_NAME" part + sudo_or_exit + case $2 in + all) install_all ;; + user) assert_user ;; + rules) + rst_title "Re-Install filtron rules" + echo + install_template --no-eval "$FILTRON_RULES" root root 644 + systemd_restart_service "${SERVICE_NAME}" + ;; + *) usage "$_usage"; exit 42;; + esac ;; + update) + sudo_or_exit + case $2 in + filtron) update_filtron ;; + *) usage "$_usage"; exit 42;; + esac ;; + remove) + sudo_or_exit + case $2 in + all) remove_all;; + user) drop_service_account "${SERVICE_USER}" ;; + *) usage "$_usage"; exit 42;; + esac ;; + activate) + sudo_or_exit + case $2 in + service) systemd_activate_service "${SERVICE_NAME}" ;; + *) usage "$_usage"; exit 42;; + esac ;; + deactivate) + sudo_or_exit + case $2 in + service) systemd_deactivate_service "${SERVICE_NAME}" ;; + *) usage "$_usage"; exit 42;; + esac ;; + apache) + sudo_or_exit + case $2 in + install) install_apache_site ;; + remove) remove_apache_site ;; + *) usage "$_usage"; exit 42;; + esac ;; + nginx) + sudo_or_exit + case $2 in + install) install_nginx_site ;; + remove) remove_nginx_site ;; + *) usage "$_usage"; exit 42;; + esac ;; + option) + sudo_or_exit + case $2 in + debug-on) echo; enable_debug ;; + debug-off) echo; disable_debug ;; + *) usage "$_usage"; exit 42;; + esac ;; + doc) rst-doc ;; + *) usage "unknown or missing command $1"; exit 42;; + esac +} + +install_all() { + rst_title "Install $SERVICE_NAME (service)" + assert_user + wait_key + install_go "${GO_PKG_URL}" "${GO_TAR}" "${SERVICE_USER}" + wait_key + install_filtron + wait_key + systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" + wait_key + echo + if ! service_is_available "http://${FILTRON_LISTEN}" ; then + err_msg "Filtron does not listening on: http://${FILTRON_LISTEN}" + fi + if apache_is_installed; then + info_msg "Apache is installed on this host." + if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then + install_apache_site + fi + elif nginx_is_installed; then + info_msg "nginx is installed on this host." + if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then + install_nginx_site + fi + fi + if ask_yn "Do you want to inspect the installation?" Ny; then + inspect_service + fi + +} + +remove_all() { + rst_title "De-Install $SERVICE_NAME (service)" + + rst_para "\ +It goes without saying that this script can only be used to remove +installations that were installed with this script." + + if ! systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then + return 42 + fi + drop_service_account "${SERVICE_USER}" + rm -r "$FILTRON_ETC" 2>&1 | prefix_stdout + if service_is_available "${PUBLIC_URL}"; then + MSG="** Don't forget to remove your public site! (${PUBLIC_URL}) **" wait_key 10 + fi +} + +assert_user() { + rst_title "user $SERVICE_USER" section + echo + tee_stderr 1 <<EOF | bash | prefix_stdout +useradd --shell /bin/bash --system \ + --home-dir "$SERVICE_HOME" \ + --comment 'Reverse HTTP proxy to filter requests' $SERVICE_USER +mkdir "$SERVICE_HOME" +chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME" +groups $SERVICE_USER +EOF + SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)" + export SERVICE_HOME + echo "export SERVICE_HOME=$SERVICE_HOME" + + cat > "$GO_ENV" <<EOF +export GOPATH=\$HOME/go-apps +export PATH=\$PATH:\$HOME/local/go/bin:\$GOPATH/bin +EOF + echo "Environment $GO_ENV has been setup." + + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" +grep -qFs -- 'source $GO_ENV' ~/.profile || echo 'source $GO_ENV' >> ~/.profile +EOF +} + +filtron_is_installed() { + [[ -f $SERVICE_HOME/go-apps/bin/filtron ]] +} + +_svcpr=" ${_Yellow}|${SERVICE_USER}|${_creset} " + +install_filtron() { + rst_title "Install filtron in user's ~/go-apps" section + echo + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr" +go get -v -u github.com/asciimoo/filtron +EOF + install_template --no-eval "$FILTRON_RULES" root root 644 +} + +update_filtron() { + rst_title "Update filtron" section + echo + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr" +go get -v -u github.com/asciimoo/filtron +EOF +} + +inspect_service() { + + rst_title "service status & log" + + cat <<EOF + +sourced ${DOT_CONFIG#"$REPO_ROOT/"} : + + PUBLIC_URL : ${PUBLIC_URL} + PUBLIC_HOST : ${PUBLIC_HOST} + FILTRON_URL_PATH : ${FILTRON_URL_PATH} + FILTRON_API : ${FILTRON_API} + FILTRON_LISTEN : ${FILTRON_LISTEN} + FILTRON_TARGET : ${FILTRON_TARGET} + +EOF + + if service_account_is_available "$SERVICE_USER"; then + info_msg "service account $SERVICE_USER available." + else + err_msg "service account $SERVICE_USER not available!" + fi + if go_is_available "$SERVICE_USER"; then + info_msg "~$SERVICE_USER: go is installed" + else + err_msg "~$SERVICE_USER: go is not installed" + fi + if filtron_is_installed; then + info_msg "~$SERVICE_USER: filtron app is installed" + else + err_msg "~$SERVICE_USER: filtron app is not installed!" + fi + + if ! service_is_available "http://${FILTRON_API}"; then + err_msg "API not available at: http://${FILTRON_API}" + fi + + if ! service_is_available "http://${FILTRON_LISTEN}" ; then + err_msg "Filtron does not listening on: http://${FILTRON_LISTEN}" + fi + + if service_is_available "http://${FILTRON_TARGET}" ; then + info_msg "Filtron's target is available at: http://${FILTRON_TARGET}" + fi + + if ! service_is_available "${PUBLIC_URL}"; then + warn_msg "Public service at ${PUBLIC_URL} is not available!" + if ! in_container; then + warn_msg "Check if public name is correct and routed or use the public IP from above." + fi + fi + + if in_container; then + lxc_suite_info + else + info_msg "public URL --> ${PUBLIC_URL}" + info_msg "internal URL --> http://${FILTRON_LISTEN}" + fi + + + local _debug_on + if ask_yn "Enable filtron debug mode?"; then + enable_debug + _debug_on=1 + fi + echo + systemctl --no-pager -l status "${SERVICE_NAME}" + echo + + info_msg "public URL --> ${PUBLIC_URL}" + # shellcheck disable=SC2059 + printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log" + read -r -s -n1 -t 5 + echo + while true; do + trap break 2 + journalctl -f -u "${SERVICE_NAME}" + done + + if [[ $_debug_on == 1 ]]; then + disable_debug + fi + return 0 +} + + +enable_debug() { + info_msg "try to enable debug mode ..." + python <<EOF +import sys, json + +debug = { + u'name': u'debug request' + , u'filters': [] + , u'interval': 0 + , u'limit': 0 + , u'actions': [{u'name': u'log'}] +} + +with open('$FILTRON_RULES') as rules: + j = json.load(rules) + +pos = None +for i in range(len(j)): + if j[i].get('name') == 'debug request': + pos = i + break +if pos is not None: + j[pos] = debug +else: + j.append(debug) +with open('$FILTRON_RULES', 'w') as rules: + json.dump(j, rules, indent=2, sort_keys=True) + +EOF + systemctl restart "${SERVICE_NAME}.service" +} + +disable_debug() { + info_msg "try to disable debug mode ..." + python <<EOF +import sys, json +with open('$FILTRON_RULES') as rules: + j = json.load(rules) + +pos = None +for i in range(len(j)): + if j[i].get('name') == 'debug request': + pos = i + break +if pos is not None: + del j[pos] + with open('$FILTRON_RULES', 'w') as rules: + json.dump(j, rules, indent=2, sort_keys=True) +EOF + systemctl restart "${SERVICE_NAME}.service" +} + +install_apache_site() { + + rst_title "Install Apache site $APACHE_FILTRON_SITE" + + rst_para "\ +This installs a reverse proxy (ProxyPass) into apache site (${APACHE_FILTRON_SITE})" + + ! apache_is_installed && info_msg "Apache is not installed." + + if ! ask_yn "Do you really want to continue?" Yn; then + return + else + install_apache + fi + + "${REPO_ROOT}/utils/searx.sh" install uwsgi + + apache_install_site --variant=filtron "${APACHE_FILTRON_SITE}" + + info_msg "testing public url .." + if ! service_is_available "${PUBLIC_URL}"; then + err_msg "Public service at ${PUBLIC_URL} is not available!" + fi +} + +remove_apache_site() { + + rst_title "Remove Apache site $APACHE_FILTRON_SITE" + + rst_para "\ +This removes apache site ${APACHE_FILTRON_SITE}." + + ! apache_is_installed && err_msg "Apache is not installed." + + if ! ask_yn "Do you really want to continue?" Yn; then + return + fi + + apache_remove_site "$APACHE_FILTRON_SITE" + +} + +install_nginx_site() { + + rst_title "Install nginx site $NGINX_FILTRON_SITE" + + rst_para "\ +This installs a reverse proxy (ProxyPass) into nginx site (${NGINX_FILTRON_SITE})" + + ! nginx_is_installed && info_msg "nginx is not installed." + + if ! ask_yn "Do you really want to continue?" Yn; then + return + else + install_nginx + fi + + "${REPO_ROOT}/utils/searx.sh" install uwsgi + + # shellcheck disable=SC2034 + SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC) + # shellcheck disable=SC2034 + SEARX_URL_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_URL_PATH) + nginx_install_app --variant=filtron "${NGINX_FILTRON_SITE}" + + info_msg "testing public url .." + if ! service_is_available "${PUBLIC_URL}"; then + err_msg "Public service at ${PUBLIC_URL} is not available!" + fi +} + +remove_nginx_site() { + + rst_title "Remove nginx site $NGINX_FILTRON_SITE" + + rst_para "\ +This removes nginx site ${NGINX_FILTRON_SITE}." + + ! nginx_is_installed && err_msg "nginx is not installed." + + if ! ask_yn "Do you really want to continue?" Yn; then + return + fi + + nginx_remove_site "$FILTRON_FILTRON_SITE" + +} + + +rst-doc() { + + eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/filtron.rst")\"" + + echo -e "\n.. START install systemd unit" + cat <<EOF +.. tabs:: + + .. group-tab:: systemd + + .. code:: bash + +EOF + eval "echo \"$(< "${TEMPLATES}/${SERVICE_SYSTEMD_UNIT}")\"" | prefix_stdout " " + echo -e "\n.. END install systemd unit" + + # for DIST_NAME in ubuntu-20.04 arch fedora; do + # ( + # DIST_ID=${DIST_NAME%-*} + # DIST_VERS=${DIST_NAME#*-} + # [[ $DIST_VERS =~ $DIST_ID ]] && DIST_VERS= + # # ... + # ) + # done +} + +# ---------------------------------------------------------------------------- +main "$@" +# ---------------------------------------------------------------------------- diff --git a/utils/lib.sh b/utils/lib.sh new file mode 100755 index 000000000..922227a89 --- /dev/null +++ b/utils/lib.sh @@ -0,0 +1,1519 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- +# SPDX-License-Identifier: AGPL-3.0-or-later +# shellcheck disable=SC2059,SC1117 + +# ubuntu, debian, arch, fedora ... +DIST_ID=$(source /etc/os-release; echo "$ID"); +# shellcheck disable=SC2034 +DIST_VERS=$(source /etc/os-release; echo "$VERSION_ID"); + +ADMIN_NAME="${ADMIN_NAME:-$(git config user.name)}" +ADMIN_NAME="${ADMIN_NAME:-$USER}" + +ADMIN_EMAIL="${ADMIN_EMAIL:-$(git config user.email)}" +ADMIN_EMAIL="${ADMIN_EMAIL:-$USER@$(hostname)}" + +if [[ -z "${REPO_ROOT}" ]]; then + REPO_ROOT=$(dirname "${BASH_SOURCE[0]}") + while [ -h "${REPO_ROOT}" ] ; do + REPO_ROOT=$(readlink "${REPO_ROOT}") + done + REPO_ROOT=$(cd "${REPO_ROOT}/.." && pwd -P ) +fi + +if [[ -z ${TEMPLATES} ]]; then + TEMPLATES="${REPO_ROOT}/utils/templates" +fi + +if [[ -z "$CACHE" ]]; then + CACHE="${REPO_ROOT}/cache" +fi + +if [[ -z ${DIFF_CMD} ]]; then + DIFF_CMD="diff -u" + if command -v colordiff >/dev/null; then + DIFF_CMD="colordiff -u" + fi +fi + +DOT_CONFIG="${DOT_CONFIG:-${REPO_ROOT}/.config.sh}" + +source_dot_config() { + if [[ ! -e "${DOT_CONFIG}" ]]; then + err_msg "configuration does not extsts at: ${DOT_CONFIG}" + return 42 + fi + # shellcheck disable=SC1090 + source "${DOT_CONFIG}" +} + +sudo_or_exit() { + # usage: sudo_or_exit + + if [ ! "$(id -u)" -eq 0 ]; then + err_msg "this command requires root (sudo) privilege!" >&2 + exit 42 + fi +} + +required_commands() { + + # usage: required_commands [cmd1 ...] + + local exit_val=0 + while [ -n "$1" ]; do + + if ! command -v "$1" &>/dev/null; then + err_msg "missing command $1" + exit_val=42 + fi + shift + done + return $exit_val +} + +# colors +# ------ + +# shellcheck disable=SC2034 +set_terminal_colors() { + _colors=8 + _creset='\e[0m' # reset all attributes + + _Black='\e[0;30m' + _White='\e[1;37m' + _Red='\e[0;31m' + _Green='\e[0;32m' + _Yellow='\e[0;33m' + _Blue='\e[0;34m' + _Violet='\e[0;35m' + _Cyan='\e[0;36m' + + _BBlack='\e[1;30m' + _BWhite='\e[1;37m' + _BRed='\e[1;31m' + _BGreen='\e[1;32m' + _BYellow='\e[1;33m' + _BBlue='\e[1;34m' + _BPurple='\e[1;35m' + _BCyan='\e[1;36m' +} + +if [ ! -p /dev/stdout ]; then + set_terminal_colors +fi + +# reST +# ---- + +if command -v fmt >/dev/null; then + export FMT="fmt -u" +else + export FMT="cat" +fi + +rst_title() { + # usage: rst_title <header-text> [part|chapter|section] + + case ${2-chapter} in + part) printf "\n${_BGreen}${1//?/=}${_creset}\n${_BCyan}${1}${_creset}\n${_BGreen}${1//?/=}${_creset}\n";; + chapter) printf "\n${_BCyan}${1}${_creset}\n${_BGreen}${1//?/=}${_creset}\n";; + section) printf "\n${_BCyan}${1}${_creset}\n${_BGreen}${1//?/-}${_creset}\n";; + *) + err_msg "invalid argument '${2}' in line $(caller)" + return 42 + ;; + esac +} + +rst_para() { + # usage: RST_INDENT=1 rst_para "lorem ipsum ..." + local prefix='' + if [[ -n $RST_INDENT ]] && [[ $RST_INDENT -gt 0 ]]; then + prefix="$(for i in $(seq 1 "$RST_INDENT"); do printf " "; done)" + echo -en "\n$*\n" | $FMT | prefix_stdout "$prefix" + else + echo -en "\n$*\n" | $FMT + fi +} + +die() { + echo -e "${_BRed}ERROR:${_creset} ${BASH_SOURCE[1]}: line ${BASH_LINENO[0]}: ${2-died ${1-1}}" >&2; + exit "${1-1}" +} + +die_caller() { + echo -e "${_BRed}ERROR:${_creset} ${BASH_SOURCE[2]}: line ${BASH_LINENO[1]}: ${FUNCNAME[1]}(): ${2-died ${1-1}}" >&2; + exit "${1-1}" +} + +err_msg() { echo -e "${_BRed}ERROR:${_creset} $*" >&2; } +warn_msg() { echo -e "${_BBlue}WARN:${_creset} $*" >&2; } +info_msg() { echo -e "${_BYellow}INFO:${_creset} $*" >&2; } + +clean_stdin() { + if [[ $(uname -s) != 'Darwin' ]]; then + while read -r -n1 -t 0.1; do : ; done + fi +} + +wait_key(){ + # usage: waitKEY [<timeout in sec>] + + clean_stdin + local _t=$1 + local msg="${MSG}" + [[ -z "$msg" ]] && msg="${_Green}** press any [${_BCyan}KEY${_Green}] to continue **${_creset}" + + [[ -n $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT + [[ -n $_t ]] && _t="-t $_t" + printf "$msg" + # shellcheck disable=SC2086 + read -r -s -n1 $_t + echo + clean_stdin +} + +ask_yn() { + # usage: ask_yn <prompt-text> [Ny|Yn] [<timeout in sec>] + + local EXIT_YES=0 # exit status 0 --> successful + local EXIT_NO=1 # exit status 1 --> error code + + local _t=$3 + [[ -n $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT + [[ -n $_t ]] && _t="-t $_t" + case "${FORCE_SELECTION:-${2}}" in + Y) return ${EXIT_YES} ;; + N) return ${EXIT_NO} ;; + Yn) + local exit_val=${EXIT_YES} + local choice="[${_BGreen}YES${_creset}/no]" + local default="Yes" + ;; + *) + local exit_val=${EXIT_NO} + local choice="[${_BGreen}NO${_creset}/yes]" + local default="No" + ;; + esac + echo + while true; do + clean_stdin + printf "$1 ${choice} " + # shellcheck disable=SC2086 + read -r -n1 $_t + if [[ -z $REPLY ]]; then + printf "$default\n"; break + elif [[ $REPLY =~ ^[Yy]$ ]]; then + exit_val=${EXIT_YES} + printf "\n" + break + elif [[ $REPLY =~ ^[Nn]$ ]]; then + exit_val=${EXIT_NO} + printf "\n" + break + fi + _t="" + err_msg "invalid choice" + done + clean_stdin + return $exit_val +} + +tee_stderr () { + + # usage:: + # tee_stderr 1 <<EOF | python -i + # print("hello") + # EOF + # ... + # >>> print("hello") + # hello + + local _t="0"; + if [[ -n $1 ]] ; then _t="$1"; fi + + (while read -r line; do + # shellcheck disable=SC2086 + sleep $_t + echo -e "$line" >&2 + echo "$line" + done) +} + +prefix_stdout () { + # usage: <cmd> | prefix_stdout [prefix] + + local prefix="${_BYellow}-->|${_creset}" + + if [[ -n $1 ]] ; then prefix="$1"; fi + + # shellcheck disable=SC2162 + (while IFS= read line; do + echo -e "${prefix}$line" + done) +} + +append_line() { + + # usage: append_line <line> <file> + # + # Append line if not exists, create file if not exists. E.g:: + # + # append_line 'source ~/.foo' ~/bashrc + + local LINE=$1 + local FILE=$2 + grep -qFs -- "$LINE" "$FILE" || echo "$LINE" >> "$FILE" +} + +cache_download() { + + # usage: cache_download <url> <local-filename> + + local exit_value=0 + + if [[ -n ${SUDO_USER} ]]; then + sudo -u "${SUDO_USER}" mkdir -p "${CACHE}" + else + mkdir -p "${CACHE}" + fi + + if [[ -f "${CACHE}/$2" ]] ; then + info_msg "already cached: $1" + info_msg " --> ${CACHE}/$2" + fi + + if [[ ! -f "${CACHE}/$2" ]]; then + info_msg "caching: $1" + info_msg " --> ${CACHE}/$2" + if [[ -n ${SUDO_USER} ]]; then + sudo -u "${SUDO_USER}" wget --progress=bar -O "${CACHE}/$2" "$1" ; exit_value=$? + else + wget --progress=bar -O "${CACHE}/$2" "$1" ; exit_value=$? + fi + if [[ ! $exit_value = 0 ]]; then + err_msg "failed to download: $1" + fi + fi +} + +backup_file() { + + # usage: backup_file /path/to/file.foo + + local stamp + stamp=$(date +"_%Y%m%d_%H%M%S") + info_msg "create backup: ${1}${stamp}" + cp -a "${1}" "${1}${stamp}" +} + +choose_one() { + + # usage: + # + # DEFAULT_SELECT= 2 \ + # choose_one <name> "your selection?" "Coffee" "Coffee with milk" + + local default=${DEFAULT_SELECT-1} + local REPLY + local env_name=$1 && shift + local choice=$1; + local max="${#@}" + local _t + [[ -n $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT + [[ -n $_t ]] && _t="-t $_t" + + list=("$@") + echo -e "${_BGreen}Menu::${_creset}" + for ((i=1; i<= $((max -1)); i++)); do + if [[ "$i" == "$default" ]]; then + echo -e " ${_BGreen}$i.${_creset}) ${list[$i]} [default]" + else + echo -e " $i.) ${list[$i]}" + fi + done + while true; do + clean_stdin + printf "$1 [${_BGreen}$default${_creset}] " + + if (( 10 > max )); then + # shellcheck disable=SC2086 + read -r -n1 $_t + else + # shellcheck disable=SC2086,SC2229 + read -r $_t + fi + # selection fits + [[ $REPLY =~ ^-?[0-9]+$ ]] && (( REPLY > 0 )) && (( REPLY < max )) && break + + # take default + [[ -z $REPLY ]] && REPLY=$default && break + + _t="" + err_msg "invalid choice" + done + eval "$env_name"='${list[${REPLY}]}' + echo + clean_stdin +} + +install_template() { + + # usage: + # + # install_template [--no-eval] [--variant=<name>] \ + # {file} [{owner} [{group} [{chmod}]]] + # + # E.g. the origin of variant 'raw' of /etc/updatedb.conf is:: + # + # ${TEMPLATES}/etc/updatedb.conf:raw + # + # To install variant 'raw' of /etc/updatedb.conf without evaluated + # replacements you can use:: + # + # install_template --variant=raw --no-eval \ + # /etc/updatedb.conf root root 644 + + local _reply="" + local do_eval=1 + local variant="" + local pos_args=("$0") + + for i in "$@"; do + case $i in + --no-eval) do_eval=0; shift ;; + --variant=*) variant=":${i#*=}"; shift ;; + *) pos_args+=("$i") ;; + esac + done + + local dst="${pos_args[1]}" + local template_origin="${TEMPLATES}${dst}${variant}" + local template_file="${TEMPLATES}${dst}" + + local owner="${pos_args[2]-$(id -un)}" + local group="${pos_args[3]-$(id -gn)}" + local chmod="${pos_args[4]-644}" + + info_msg "install (eval=$do_eval): ${dst}" + [[ -n $variant ]] && info_msg "variant --> ${variant}" + + if [[ ! -f "${template_origin}" ]] ; then + err_msg "${template_origin} does not exists" + err_msg "... can't install $dst" + wait_key 30 + return 42 + fi + + if [[ "$do_eval" == "1" ]]; then + template_file="${CACHE}${dst}${variant}" + info_msg "BUILD template ${template_file}" + if [[ -n ${SUDO_USER} ]]; then + sudo -u "${SUDO_USER}" mkdir -p "$(dirname "${template_file}")" + else + mkdir -p "$(dirname "${template_file}")" + fi + # shellcheck disable=SC2086 + eval "echo \"$(cat ${template_origin})\"" > "${template_file}" + if [[ -n ${SUDO_USER} ]]; then + chown "${SUDO_USER}:${SUDO_USER}" "${template_file}" + fi + else + template_file=$template_origin + fi + + mkdir -p "$(dirname "${dst}")" + + if [[ ! -f "${dst}" ]]; then + info_msg "install: ${template_file}" + sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \ + "${template_file}" "${dst}" | prefix_stdout + return $? + fi + + if [[ -f "${dst}" ]] && cmp --silent "${template_file}" "${dst}" ; then + info_msg "file ${dst} allready installed" + return 0 + fi + + info_msg "diffrent file ${dst} allready exists on this host" + + while true; do + choose_one _reply "choose next step with file $dst" \ + "replace file" \ + "leave file unchanged" \ + "interactiv shell" \ + "diff files" + + case $_reply in + "replace file") + info_msg "install: ${template_file}" + sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \ + "${template_file}" "${dst}" | prefix_stdout + break + ;; + "leave file unchanged") + break + ;; + "interactiv shell") + echo -e "// edit ${_Red}${dst}${_creset} to your needs" + echo -e "// exit with [${_BCyan}CTRL-D${_creset}]" + sudo -H -u "${owner}" -i + $DIFF_CMD "${dst}" "${template_file}" + echo + echo -e "// ${_BBlack}did you edit file ...${_creset}" + echo -en "// ${_Red}${dst}${_creset}" + if ask_yn "//${_BBlack}... to your needs?${_creset}"; then + break + fi + ;; + "diff files") + $DIFF_CMD "${dst}" "${template_file}" | prefix_stdout + esac + done +} + + +service_is_available() { + + # usage: service_is_available <URL> + + [[ -z $1 ]] && die_caller 42 "missing argument <URL>" + local URL="$1" + http_code=$(curl -H 'Cache-Control: no-cache' \ + --silent -o /dev/null --head --write-out '%{http_code}' --insecure \ + "${URL}") + exit_val=$? + if [[ $exit_val = 0 ]]; then + info_msg "got $http_code from ${URL}" + fi + case "$http_code" in + 404|410|423) exit_val=$http_code;; + esac + return "$exit_val" +} + +# golang +# ------ + +go_is_available() { + + # usage: go_is_available $SERVICE_USER && echo "go is installed!" + + sudo -i -u "${1}" which go &>/dev/null +} + +install_go() { + + # usage: install_go "${GO_PKG_URL}" "${GO_TAR}" "${SERVICE_USER}" + + local _svcpr=" ${_Yellow}|${3}|${_creset} " + + rst_title "Install Go in user's HOME" section + + rst_para "download and install go binary .." + cache_download "${1}" "${2}" + + tee_stderr 0.1 <<EOF | sudo -i -u "${3}" | prefix_stdout "$_svcpr" +echo \$PATH +echo \$GOPATH +mkdir -p \$HOME/local +rm -rf \$HOME/local/go +tar -C \$HOME/local -xzf ${CACHE}/${2} +EOF + sudo -i -u "${3}" <<EOF | prefix_stdout +! which go >/dev/null && echo "ERROR - Go Installation not found in PATH!?!" +which go >/dev/null && go version && echo "congratulations -- Go installation OK :)" +EOF +} + +# system accounts +# --------------- + +service_account_is_available() { + + # usage: service_account_is_available "$SERVICE_USER" && echo "OK" + + sudo -i -u "$1" echo \$HOME &>/dev/null +} + +drop_service_account() { + + # usage: drop_service_account "${SERVICE_USER}" + + rst_title "Drop ${1} HOME" section + if ask_yn "Do you really want to drop ${1} home folder?"; then + userdel -r -f "${1}" 2>&1 | prefix_stdout + else + rst_para "Leave HOME folder $(du -sh "${1}") unchanged." + fi +} + +interactive_shell(){ + + # usage: interactive_shell "${SERVICE_USER}" + + echo -e "// exit with [${_BCyan}CTRL-D${_creset}]" + sudo -H -u "${1}" -i +} + + +# systemd +# ------- + +SYSTEMD_UNITS="${SYSTEMD_UNITS:-/lib/systemd/system}" + +systemd_install_service() { + + # usage: systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" + + rst_title "Install System-D Unit ${1}" section + echo + install_template "${2}" root root 644 + wait_key + systemd_activate_service "${1}" +} + +systemd_remove_service() { + + # usage: systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" + + if ! ask_yn "Do you really want to deinstall systemd unit ${1}?"; then + return 42 + fi + systemd_deactivate_service "${1}" + rm "${2}" 2>&1 | prefix_stdout +} + +systemd_activate_service() { + + # usage: systemd_activate_service "${SERVICE_NAME}" + + rst_title "Activate ${1} (service)" section + echo + tee_stderr <<EOF | bash 2>&1 +systemctl enable ${1}.service +systemctl restart ${1}.service +EOF + tee_stderr <<EOF | bash 2>&1 +systemctl status --no-pager ${1}.service +EOF +} + +systemd_deactivate_service() { + + # usage: systemd_deactivate_service "${SERVICE_NAME}" + + rst_title "De-Activate ${1} (service)" section + echo + tee_stderr <<EOF | bash 2>&1 | prefix_stdout +systemctl stop ${1}.service +systemctl disable ${1}.service +EOF +} + +systemd_restart_service() { + + # usage: systemd_restart_service "${SERVICE_NAME}" + + rst_title "Restart ${1} (service)" section + echo + tee_stderr <<EOF | bash 2>&1 +systemctl restart ${1}.service +EOF + tee_stderr <<EOF | bash 2>&1 +systemctl status --no-pager ${1}.service +EOF +} + + +# nginx +# ----- + +nginx_distro_setup() { + # shellcheck disable=SC2034 + + NGINX_DEFAULT_SERVER=/etc/nginx/nginx.conf + + # Including *location* directives from a dedicated config-folder into the + # server directive is, what what fedora (already) does. + NGINX_APPS_ENABLED="/etc/nginx/default.d" + + # We add a apps-available folder and linking configurations into the + # NGINX_APPS_ENABLED folder. See also nginx_include_apps_enabled(). + NGINX_APPS_AVAILABLE="/etc/nginx/default.apps-available" + + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) + NGINX_PACKAGES="nginx" + NGINX_DEFAULT_SERVER=/etc/nginx/sites-available/default + ;; + arch-*) + NGINX_PACKAGES="nginx-mainline" + ;; + fedora-*) + NGINX_PACKAGES="nginx" + ;; + *) + err_msg "$DIST_ID-$DIST_VERS: nginx not yet implemented" + ;; + esac +} +nginx_distro_setup + +install_nginx(){ + info_msg "installing nginx ..." + pkg_install "${NGINX_PACKAGES}" + case $DIST_ID-$DIST_VERS in + arch-*|fedora-*) + systemctl enable nginx + systemctl start nginx + ;; + esac +} + +nginx_is_installed() { + command -v nginx &>/dev/null +} + +nginx_reload() { + + info_msg "reload nginx .." + echo + if ! nginx -t; then + err_msg "testing nginx configuration failed" + return 42 + fi + systemctl restart nginx +} + +nginx_install_app() { + + # usage: nginx_install_app [<template option> ...] <myapp> + # + # <template option>: see install_template + + local template_opts=() + local pos_args=("$0") + + for i in "$@"; do + case $i in + -*) template_opts+=("$i");; + *) pos_args+=("$i");; + esac + done + + nginx_include_apps_enabled "${NGINX_DEFAULT_SERVER}" + + install_template "${template_opts[@]}" \ + "${NGINX_APPS_AVAILABLE}/${pos_args[1]}" \ + root root 644 + nginx_enable_app "${pos_args[1]}" + info_msg "installed nginx app: ${pos_args[1]}" +} + +nginx_include_apps_enabled() { + + # Add the *NGINX_APPS_ENABLED* infrastruture to a nginx server block. Such + # infrastruture is already known from fedora, including apps (location + # directives) from the /etc/nginx/default.d folder into the *default* nginx + # server. + + # usage: nginx_include_apps_enabled <config-file> + # + # config-file: Config file with server directive in. + + [[ -z $1 ]] && die_caller 42 "missing argument <config-file>" + local server_conf="$1" + + # include /etc/nginx/default.d/*.conf; + local include_directive="include ${NGINX_APPS_ENABLED}/*.conf;" + local include_directive_re="^\s*include ${NGINX_APPS_ENABLED}/\*\.conf;" + + info_msg "checking existence: '${include_directive}' in file ${server_conf}" + if grep "${include_directive_re}" "${server_conf}"; then + info_msg "OK, already exists." + return + fi + + info_msg "add missing directive: '${include_directive}'" + cp "${server_conf}" "${server_conf}.bak" + + ( + local line + local stage=0 + while IFS= read -r line + do + echo "$line" + if [[ $stage = 0 ]]; then + if [[ $line =~ ^[[:space:]]*server*[[:space:]]*\{ ]]; then + stage=1 + fi + fi + + if [[ $stage = 1 ]]; then + echo " # Load configuration files for the default server block." + echo " $include_directive" + echo "" + stage=2 + fi + done < "${server_conf}.bak" + ) > "${server_conf}" + +} + +nginx_remove_app() { + + # usage: nginx_remove_app <myapp.conf> + + info_msg "remove nginx app: $1" + nginx_dissable_app "$1" + rm -f "${NGINX_APPS_AVAILABLE}/$1" +} + +nginx_enable_app() { + + # usage: nginx_enable_app <myapp.conf> + + local CONF="$1" + + info_msg "enable nginx app: ${CONF}" + mkdir -p "${NGINX_APPS_ENABLED}" + rm -f "${NGINX_APPS_ENABLED}/${CONF}" + ln -s "${NGINX_APPS_AVAILABLE}/${CONF}" "${NGINX_APPS_ENABLED}/${CONF}" + nginx_reload +} + +nginx_dissable_app() { + + # usage: nginx_disable_app <myapp.conf> + + local CONF="$1" + + info_msg "disable nginx app: ${CONF}" + rm -f "${NGINX_APPS_ENABLED}/${CONF}" + nginx_reload +} + + +# Apache +# ------ + +apache_distro_setup() { + # shellcheck disable=SC2034 + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) + # debian uses the /etc/apache2 path, while other distros use + # the apache default at /etc/httpd + APACHE_SITES_AVAILABLE="/etc/apache2/sites-available" + APACHE_SITES_ENABLED="/etc/apache2/sites-enabled" + APACHE_MODULES="/usr/lib/apache2/modules" + APACHE_PACKAGES="apache2" + ;; + arch-*) + APACHE_SITES_AVAILABLE="/etc/httpd/sites-available" + APACHE_SITES_ENABLED="/etc/httpd/sites-enabled" + APACHE_MODULES="modules" + APACHE_PACKAGES="apache" + ;; + fedora-*) + APACHE_SITES_AVAILABLE="/etc/httpd/sites-available" + APACHE_SITES_ENABLED="/etc/httpd/sites-enabled" + APACHE_MODULES="modules" + APACHE_PACKAGES="httpd" + ;; + *) + err_msg "$DIST_ID-$DIST_VERS: apache not yet implemented" + ;; + esac +} + +apache_distro_setup + +install_apache(){ + info_msg "installing apache ..." + pkg_install "$APACHE_PACKAGES" + case $DIST_ID-$DIST_VERS in + arch-*|fedora-*) + if ! grep "IncludeOptional sites-enabled" "/etc/httpd/conf/httpd.conf"; then + echo "IncludeOptional sites-enabled/*.conf" >> "/etc/httpd/conf/httpd.conf" + fi + systemctl enable httpd + systemctl start httpd + ;; + esac +} + +apache_is_installed() { + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) (command -v apachectl) &>/dev/null;; + arch-*) (command -v httpd) &>/dev/null;; + fedora-*) (command -v httpd) &>/dev/null;; + esac +} + +apache_reload() { + + info_msg "reload apache .." + echo + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) + sudo -H apachectl configtest + sudo -H systemctl force-reload apache2 + ;; + arch-*| fedora-*) + sudo -H httpd -t + sudo -H systemctl force-reload httpd + ;; + esac +} + +apache_install_site() { + + # usage: apache_install_site [<template option> ...] <mysite.conf> + # + # <template option>: see install_template + + local template_opts=() + local pos_args=("$0") + + for i in "$@"; do + case $i in + -*) template_opts+=("$i");; + *) pos_args+=("$i");; + esac + done + + install_template "${template_opts[@]}" \ + "${APACHE_SITES_AVAILABLE}/${pos_args[1]}" \ + root root 644 + apache_enable_site "${pos_args[1]}" + info_msg "installed apache site: ${pos_args[1]}" +} + +apache_remove_site() { + + # usage: apache_remove_site <mysite.conf> + + info_msg "remove apache site: $1" + apache_dissable_site "$1" + rm -f "${APACHE_SITES_AVAILABLE}/$1" +} + +apache_enable_site() { + + # usage: apache_enable_site <mysite.conf> + + local CONF="$1" + + info_msg "enable apache site: ${CONF}" + + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) + sudo -H a2ensite -q "${CONF}" + ;; + arch-*) + mkdir -p "${APACHE_SITES_ENABLED}" + rm -f "${APACHE_SITES_ENABLED}/${CONF}" + ln -s "${APACHE_SITES_AVAILABLE}/${CONF}" "${APACHE_SITES_ENABLED}/${CONF}" + ;; + fedora-*) + mkdir -p "${APACHE_SITES_ENABLED}" + rm -f "${APACHE_SITES_ENABLED}/${CONF}" + ln -s "${APACHE_SITES_AVAILABLE}/${CONF}" "${APACHE_SITES_ENABLED}/${CONF}" + ;; + esac + apache_reload +} + +apache_dissable_site() { + + # usage: apache_disable_site <mysite.conf> + + local CONF="$1" + + info_msg "disable apache site: ${CONF}" + + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) + sudo -H a2dissite -q "${CONF}" + ;; + arch-*) + rm -f "${APACHE_SITES_ENABLED}/${CONF}" + ;; + fedora-*) + rm -f "${APACHE_SITES_ENABLED}/${CONF}" + ;; + esac + apache_reload +} + +# uWSGI +# ----- + +uWSGI_SETUP="${uWSGI_SETUP:=/etc/uwsgi}" +uWSGI_USER= +uWSGI_GROUP= + +# How distros manage uWSGI apps is very different. From uWSGI POV read: +# - https://uwsgi-docs.readthedocs.io/en/latest/Management.html + +uWSGI_distro_setup() { + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) + # init.d --> /usr/share/doc/uwsgi/README.Debian.gz + # For uWSGI debian uses the LSB init process, this might be changed + # one day, see https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=833067 + uWSGI_APPS_AVAILABLE="${uWSGI_SETUP}/apps-available" + uWSGI_APPS_ENABLED="${uWSGI_SETUP}/apps-enabled" + uWSGI_PACKAGES="uwsgi" + ;; + arch-*) + # systemd --> /usr/lib/systemd/system/uwsgi@.service + # For uWSGI archlinux uses systemd template units, see + # - http://0pointer.de/blog/projects/instances.html + # - https://uwsgi-docs.readthedocs.io/en/latest/Systemd.html#one-service-per-app-in-systemd + uWSGI_APPS_AVAILABLE="${uWSGI_SETUP}/apps-archlinux" + uWSGI_APPS_ENABLED="${uWSGI_SETUP}" + uWSGI_PACKAGES="uwsgi" + ;; + fedora-*) + # systemd --> /usr/lib/systemd/system/uwsgi.service + # The unit file starts uWSGI in emperor mode (/etc/uwsgi.ini), see + # - https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html + uWSGI_APPS_AVAILABLE="${uWSGI_SETUP}/apps-available" + uWSGI_APPS_ENABLED="${uWSGI_SETUP}.d" + uWSGI_PACKAGES="uwsgi" + uWSGI_USER="uwsgi" + uWSGI_GROUP="uwsgi" + ;; + *) + err_msg "$DIST_ID-$DIST_VERS: uWSGI not yet implemented" + ;; +esac +} + +uWSGI_distro_setup + +install_uwsgi(){ + info_msg "installing uwsgi ..." + pkg_install "$uWSGI_PACKAGES" + case $DIST_ID-$DIST_VERS in + fedora-*) + # enable & start should be called once at uWSGI installation time + systemctl enable uwsgi + systemctl restart uwsgi + ;; + esac +} + +uWSGI_restart() { + + # usage: uWSGI_restart() <myapp.ini> + + local CONF="$1" + + [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>" + info_msg "restart uWSGI service" + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) + # the 'service' method seems broken in that way, that it (re-)starts + # the whole uwsgi process. + service uwsgi restart "${CONF%.*}" + ;; + arch-*) + # restart systemd template instance + if uWSGI_app_available "${CONF}"; then + systemctl restart "uwsgi@${CONF%.*}" + else + info_msg "[uWSGI:systemd-template] ${CONF} not installed (no need to restart)" + fi + ;; + fedora-*) + # in emperor mode, just touch the file to restart + if uWSGI_app_enabled "${CONF}"; then + touch "${uWSGI_APPS_ENABLED}/${CONF}" + # it seems, there is a polling time in between touch and restart + # of the service. + sleep 3 + else + info_msg "[uWSGI:emperor] ${CONF} not installed (no need to restart)" + fi + ;; + *) + err_msg "$DIST_ID-$DIST_VERS: uWSGI not yet implemented" + return 42 + ;; + esac +} + +uWSGI_prepare_app() { + + # usage: uWSGI_prepare_app <myapp.ini> + + [[ -z $1 ]] && die_caller 42 "missing argument <myapp.ini>" + + local APP="${1%.*}" + + case $DIST_ID-$DIST_VERS in + fedora-*) + # in emperor mode, the uwsgi user is the owner of the sockets + info_msg "prepare (uwsgi:uwsgi) /run/uwsgi/app/${APP}" + mkdir -p "/run/uwsgi/app/${APP}" + chown -R "uwsgi:uwsgi" "/run/uwsgi/app/${APP}" + ;; + *) + info_msg "prepare (${SERVICE_USER}:${SERVICE_GROUP}) /run/uwsgi/app/${APP}" + mkdir -p "/run/uwsgi/app/${APP}" + chown -R "${SERVICE_USER}:${SERVICE_GROUP}" "/run/uwsgi/app/${APP}" + ;; + esac +} + + +uWSGI_app_available() { + # usage: uWSGI_app_available <myapp.ini> + local CONF="$1" + + [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>" + [[ -f "${uWSGI_APPS_AVAILABLE}/${CONF}" ]] +} + +uWSGI_install_app() { + + # usage: uWSGI_install_app [<template option> ...] <myapp.ini> + # + # <template option>: see install_template + + local pos_args=("$0") + + for i in "$@"; do + case $i in + -*) template_opts+=("$i");; + *) pos_args+=("$i");; + esac + done + uWSGI_prepare_app "${pos_args[1]}" + mkdir -p "${uWSGI_APPS_AVAILABLE}" + install_template "${template_opts[@]}" \ + "${uWSGI_APPS_AVAILABLE}/${pos_args[1]}" \ + root root 644 + uWSGI_enable_app "${pos_args[1]}" + uWSGI_restart "${pos_args[1]}" + info_msg "uWSGI app: ${pos_args[1]} is installed" +} + +uWSGI_remove_app() { + + # usage: uWSGI_remove_app <myapp.ini> + + local CONF="$1" + + [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>" + info_msg "remove uWSGI app: ${CONF}" + uWSGI_disable_app "${CONF}" + uWSGI_restart "${CONF}" + rm -f "${uWSGI_APPS_AVAILABLE}/${CONF}" +} + +uWSGI_app_enabled() { + # usage: uWSGI_app_enabled <myapp.ini> + + local exit_val=0 + local CONF="$1" + + [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>" + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) + [[ -f "${uWSGI_APPS_ENABLED}/${CONF}" ]] + exit_val=$? + ;; + arch-*) + systemctl -q is-enabled "uwsgi@${CONF%.*}" + exit_val=$? + ;; + fedora-*) + [[ -f "${uWSGI_APPS_ENABLED}/${CONF}" ]] + exit_val=$? + ;; + *) + # FIXME + err_msg "$DIST_ID-$DIST_VERS: uWSGI not yet implemented" + exit_val=1 + ;; + esac + return $exit_val +} + +# shellcheck disable=SC2164 +uWSGI_enable_app() { + + # usage: uWSGI_enable_app <myapp.ini> + + local CONF="$1" + + [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>" + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) + mkdir -p "${uWSGI_APPS_ENABLED}" + rm -f "${uWSGI_APPS_ENABLED}/${CONF}" + ln -s "${uWSGI_APPS_AVAILABLE}/${CONF}" "${uWSGI_APPS_ENABLED}/${CONF}" + info_msg "enabled uWSGI app: ${CONF} (restart required)" + ;; + arch-*) + mkdir -p "${uWSGI_APPS_ENABLED}" + rm -f "${uWSGI_APPS_ENABLED}/${CONF}" + ln -s "${uWSGI_APPS_AVAILABLE}/${CONF}" "${uWSGI_APPS_ENABLED}/${CONF}" + systemctl enable "uwsgi@${CONF%.*}" + info_msg "enabled uWSGI app: ${CONF} (restart required)" + ;; + fedora-*) + mkdir -p "${uWSGI_APPS_ENABLED}" + rm -f "${uWSGI_APPS_ENABLED}/${CONF}" + ln -s "${uWSGI_APPS_AVAILABLE}/${CONF}" "${uWSGI_APPS_ENABLED}/${CONF}" + chown "${uWSGI_USER}:${uWSGI_GROUP}" "${uWSGI_APPS_ENABLED}/${CONF}" + info_msg "enabled uWSGI app: ${CONF}" + ;; + *) + # FIXME + err_msg "$DIST_ID-$DIST_VERS: uWSGI not yet implemented" + ;; + esac +} + +uWSGI_disable_app() { + + # usage: uWSGI_disable_app <myapp.ini> + + local CONF="$1" + + [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>" + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) + service uwsgi stop "${CONF%.*}" + rm -f "${uWSGI_APPS_ENABLED}/${CONF}" + info_msg "disabled uWSGI app: ${CONF} (restart uWSGI required)" + ;; + arch-*) + systemctl stop "uwsgi@${CONF%.*}" + systemctl disable "uwsgi@${CONF%.*}" + rm -f "${uWSGI_APPS_ENABLED}/${CONF}" + ;; + fedora-*) + # in emperor mode, just remove the app.ini file + rm -f "${uWSGI_APPS_ENABLED}/${CONF}" + ;; + *) + # FIXME + err_msg "$DIST_ID-$DIST_VERS: uWSGI not yet implemented" + ;; + esac +} + +# distro's package manager +# ------------------------ + +_apt_pkg_info_is_updated=0 + +pkg_install() { + + # usage: TITEL='install foobar' pkg_install foopkg barpkg + + rst_title "${TITLE:-installation of packages}" section + echo -e "\npackage(s)::\n" + # shellcheck disable=SC2068 + echo " " $@ | $FMT + + if ! ask_yn "Should packages be installed?" Yn 30; then + return 42 + fi + case $DIST_ID in + ubuntu|debian) + if [[ $_apt_pkg_info_is_updated == 0 ]]; then + export _apt_pkg_info_is_updated=1 + apt update + fi + # shellcheck disable=SC2068 + apt-get install -m -y $@ + ;; + arch) + # shellcheck disable=SC2068 + pacman -Sy --noconfirm $@ + ;; + fedora) + # shellcheck disable=SC2068 + dnf install -y $@ + ;; + esac +} + +pkg_remove() { + + # usage: TITEL='remove foobar' pkg_remove foopkg barpkg + + rst_title "${TITLE:-remove packages}" section + echo -e "\npackage(s)::\n" + # shellcheck disable=SC2068 + echo " " $@ | $FMT + + if ! ask_yn "Should packages be removed (purge)?" Yn 30; then + return 42 + fi + case $DIST_ID in + ubuntu|debian) + # shellcheck disable=SC2068 + apt-get purge --autoremove --ignore-missing -y $@ + ;; + arch) + # shellcheck disable=SC2068 + pacman -R --noconfirm $@ + ;; + fedora) + # shellcheck disable=SC2068 + dnf remove -y $@ + ;; + esac +} + +pkg_is_installed() { + + # usage: pkg_is_install foopkg || pkg_install foopkg + + case $DIST_ID in + ubuntu|debian) + dpkg -l "$1" &> /dev/null + return $? + ;; + arch) + pacman -Qsq "$1" &> /dev/null + return $? + ;; + fedora) + dnf list -q --installed "$1" &> /dev/null + return $? + ;; + esac +} + +# git tooling +# ----------- + +# shellcheck disable=SC2164 +git_clone() { + + # usage: + # + # git_clone <url> <name> [<branch> [<user>]] + # git_clone <url> <path> [<branch> [<user>]] + # + # First form uses $CACHE/<name> as destination folder, second form clones + # into <path>. If repository is allready cloned, pull from <branch> and + # update working tree (if needed, the caller has to stash local changes). + # + # git clone https://github.com/asciimoo/searx searx-src origin/master searxlogin + # + + local url="$1" + local dest="$2" + local branch="$3" + local user="$4" + local bash_cmd="bash" + local remote="origin" + + if [[ ! "${dest:0:1}" = "/" ]]; then + dest="$CACHE/$dest" + fi + + [[ -z $branch ]] && branch=master + [[ -z $user ]] && [[ -n "${SUDO_USER}" ]] && user="${SUDO_USER}" + [[ -n $user ]] && bash_cmd="sudo -H -u $user -i" + + if [[ -d "${dest}" ]] ; then + info_msg "already cloned: $dest" + tee_stderr 0.1 <<EOF | $bash_cmd 2>&1 | prefix_stdout " ${_Yellow}|$user|${_creset} " +cd "${dest}" +git checkout -m -B "$branch" --track "$remote/$branch" +git pull --all +EOF + else + info_msg "clone into: $dest" + tee_stderr 0.1 <<EOF | $bash_cmd 2>&1 | prefix_stdout " ${_Yellow}|$user|${_creset} " +mkdir -p "$(dirname "$dest")" +cd "$(dirname "$dest")" +git clone --branch "$branch" --origin "$remote" "$url" "$(basename "$dest")" +EOF + fi +} + +# containers +# ---------- + +in_container() { + # Test if shell runs in a container. + # + # usage: in_container && echo "process running inside a LXC container" + # in_container || echo "process is not running inside a LXC container" + # + # sudo_or_exit + # hint: Reads init process environment, therefore root access is required! + # to be safe, take a look at the environment of process 1 (/sbin/init) + # grep -qa 'container=lxc' /proc/1/environ + + # see lxc_init_container_env + [[ -f /.lxcenv ]] +} + +LXC_ENV_FOLDER= +if in_container; then + # shellcheck disable=SC2034 + LXC_ENV_FOLDER="lxc/$(hostname)/" +fi + +lxc_init_container_env() { + + # usage: lxc_init_container_env <name> + + # Create a /.lxcenv file in the root folder. Call this once after the + # container is inital started and before installing any boilerplate stuff. + + info_msg "create /.lxcenv in container $1" + cat <<EOF | lxc exec "${1}" -- bash | prefix_stdout "[${_BBlue}${1}${_creset}] " +touch "/.lxcenv" +ls -l "/.lxcenv" +EOF +} + +# apt packages +LXC_BASE_PACKAGES_debian="bash git build-essential python3 virtualenv" + +# pacman packages +LXC_BASE_PACKAGES_arch="bash git base-devel python python-virtualenv" + +# dnf packages +LXC_BASE_PACKAGES_fedora="bash git @development-tools python virtualenv" + +case $DIST_ID in + ubuntu|debian) LXC_BASE_PACKAGES="${LXC_BASE_PACKAGES_debian}" ;; + arch) LXC_BASE_PACKAGES="${LXC_BASE_PACKAGES_arch}" ;; + fedora) LXC_BASE_PACKAGES="${LXC_BASE_PACKAGES_fedora}" ;; + *) err_msg "$DIST_ID-$DIST_VERS: pkg_install LXC_BASE_PACKAGES not yet implemented" ;; +esac + +lxc_install_base_packages() { + info_msg "install LXC_BASE_PACKAGES in container $1" + pkg_install "${LXC_BASE_PACKAGES}" +} + + +lxc_image_copy() { + + # usage: lxc_copy_image <remote image> <local image> + # + # lxc_copy_image "images:ubuntu/19.10" "ubu1910" + + if lxc_image_exists "local:${LXC_SUITE[i+1]}"; then + info_msg "image ${LXC_SUITE[i]} already copied --> ${LXC_SUITE[i+1]}" + else + info_msg "copy image locally ${LXC_SUITE[i]} --> ${LXC_SUITE[i+1]}" + lxc image copy "${LXC_SUITE[i]}" local: \ + --alias "${LXC_SUITE[i+1]}" | prefix_stdout + fi +} + +lxc_init_container() { + + # usage: lxc_init_container <image name> <container name> + + local image_name="$1" + local container_name="$2" + + if lxc info "${container_name}" &>/dev/null; then + info_msg "container '${container_name}' already exists" + else + info_msg "create container instance: ${container_name}" + lxc init "local:${image_name}" "${container_name}" + fi +} + +lxc_exists(){ + + # usage: lxc_exists <name> || echo "container <name> does not exists" + + lxc info "$1" &>/dev/null +} + +lxc_image_exists(){ + # usage: lxc_image_exists <alias> || echo "image <alias> does locally not exists" + + lxc image info "local:$1" &>/dev/null + +} + +lxc_delete_container() { + + # usage: lxc_delete_container <container-name> + + if lxc info "$1" &>/dev/null; then + info_msg "stop & delete instance ${_BBlue}${1}${_creset}" + lxc stop "$1" &>/dev/null + lxc delete "$1" | prefix_stdout + else + warn_msg "instance '$1' does not exist / can't delete :o" + fi +} + +lxc_delete_local_image() { + + # usage: lxc_delete_local_image <container-name> + + info_msg "delete image 'local:$i'" + lxc image delete "local:$i" +} + + +# IP +# -- + +global_IPs(){ + # usage: global_IPS + # + # print list of host's SCOPE global addresses and adapters e.g:: + # + # $ global_IPs + # enp4s0|192.168.1.127 + # lxdbr0|10.246.86.1 + # lxdbr0|fd42:8c58:2cd:b73f::1 + + ip -o addr show | sed -nr 's/[0-9]*:\s*([a-z0-9]*).*inet[6]?\s*([a-z0-9.:]*).*scope global.*/\1|\2/p' +} + +primary_ip() { + + case $DIST_ID in + arch) + ip -o addr show \ + | sed -nr 's/[0-9]*:\s*([a-z0-9]*).*inet[6]?\s*([a-z0-9.:]*).*scope global.*/\2/p' \ + | head -n 1 + ;; + *) hostname -I | cut -d' ' -f1 ;; + esac +} + +# URL +# --- + +url_replace_hostname(){ + + # usage: url_replace_hostname <url> <new hostname> + + # to replace hostname by primary IP:: + # + # url_replace_hostname http://searx-ubu1604/morty $(primary_ip) + # http://10.246.86.250/morty + + # shellcheck disable=SC2001 + echo "$1" | sed "s|\(http[s]*://\)[^/]*\(.*\)|\1$2\2|" +} diff --git a/utils/lxc-searx.env b/utils/lxc-searx.env new file mode 100644 index 000000000..a51312fb8 --- /dev/null +++ b/utils/lxc-searx.env @@ -0,0 +1,95 @@ +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- +# SPDX-License-Identifier: AGPL-3.0-or-later +# shellcheck shell=bash + +# This file is a setup of a LXC suite. It is sourced from different context, do +# not manipulate the environment directly, implement functions and manipulate +# environment only is subshells! + +# ---------------------------------------------------------------------------- +# config +# ---------------------------------------------------------------------------- + +# shellcheck disable=SC2034 +LXC_SUITE_NAME="searx" +lxc_set_suite_env() { + # name of https://images.linuxcontainers.org + export LINUXCONTAINERS_ORG_NAME="${LINUXCONTAINERS_ORG_NAME:-images}" + export LXC_HOST_PREFIX="${LXC_SUITE_NAME:-searx}" + export LXC_SUITE=( + + # to disable containers, comment out lines .. + + # end of standard support see https://wiki.ubuntu.com/Releases + "$LINUXCONTAINERS_ORG_NAME:ubuntu/16.04" "ubu1604" # April 2021 + "$LINUXCONTAINERS_ORG_NAME:ubuntu/18.04" "ubu1804" # April 2023 + "$LINUXCONTAINERS_ORG_NAME:ubuntu/19.10" "ubu1910" # July 2020 + "$LINUXCONTAINERS_ORG_NAME:ubuntu/20.04" "ubu2004" # future (EOL 2030) + + # EOL see https://fedoraproject.org/wiki/Releases + "$LINUXCONTAINERS_ORG_NAME:fedora/31" "fedora31" + + # rolling releases see https://www.archlinux.org/releng/releases/ + "$LINUXCONTAINERS_ORG_NAME:archlinux" "archlinux" + ) + + PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}" + if in_container; then + # container hostnames do not have a DNS entry: use primary IP! + PUBLIC_URL="http://$(primary_ip)/searx" + + # make GUEST's services public to the HOST + FILTRON_API="0.0.0.0:4005" + FILTRON_LISTEN="0.0.0.0:4004" + MORTY_LISTEN="0.0.0.0:3000" + + # export LXC specific environment + export PUBLIC_URL FILTRON_API FILTRON_LISTEN MORTY_LISTEN + fi +} + +lxc_suite_install_info() { + ( + lxc_set_suite_env + cat <<EOF +LXC suite: ${LXC_SUITE_NAME} --> ${PUBLIC_URL} + suite includes searx, morty & filtron +suite images: +$(echo " ${LOCAL_IMAGES[*]}" | $FMT) +suite containers: +$(echo " ${CONTAINERS[*]}" | $FMT) +EOF + ) + } + +lxc_suite_install() { + ( + lxc_set_suite_env + FORCE_TIMEOUT=0 + export FORCE_TIMEOUT + "${LXC_REPO_ROOT}/utils/searx.sh" install all + "${LXC_REPO_ROOT}/utils/morty.sh" install all + "${LXC_REPO_ROOT}/utils/filtron.sh" install all + + rst_title "suite installation finished ($(hostname))" part + lxc_suite_info + echo + ) +} + +lxc_suite_info() { + ( + lxc_set_suite_env + for ip in $(global_IPs) ; do + if [[ $ip =~ .*:.* ]]; then + info_msg "(${ip%|*}) IPv6: http://[${ip#*|}]" + else + # IPv4: + # shellcheck disable=SC2034,SC2031 + info_msg "(${ip%|*}) filtron: http://${ip#*|}:4004/ $PUBLIC_URL" + info_msg "(${ip%|*}) morty: http://${ip#*|}:3000/ $PUBLIC_URL_MORTY" + info_msg "(${ip%|*}) docs-live: http://${ip#*|}:8080/" + fi + done + ) +} diff --git a/utils/lxc.sh b/utils/lxc.sh new file mode 100755 index 000000000..a0688bc07 --- /dev/null +++ b/utils/lxc.sh @@ -0,0 +1,552 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- +# SPDX-License-Identifier: AGPL-3.0-or-later + +# shellcheck source=utils/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +source_dot_config + +# load environment of the LXC suite +LXC_ENV="${LXC_ENV:-${REPO_ROOT}/utils/lxc-searx.env}" +source "$LXC_ENV" +lxc_set_suite_env + +# ---------------------------------------------------------------------------- +# config +# ---------------------------------------------------------------------------- +# +# read also: +# - https://lxd.readthedocs.io/en/latest/ + +LXC_HOST_PREFIX="${LXC_HOST_PREFIX:-test}" + +# where all folders from HOST are mounted +LXC_SHARE_FOLDER="/share" +LXC_REPO_ROOT="${LXC_SHARE_FOLDER}/$(basename "${REPO_ROOT}")" + +ubu1604_boilerplate=" +export DEBIAN_FRONTEND=noninteractive +apt-get update -y +apt-get upgrade -y +apt-get install -y git curl wget +" +ubu1804_boilerplate="$ubu1604_boilerplate" +ubu1904_boilerplate="$ubu1804_boilerplate" +ubu1910_boilerplate="$ubu1904_boilerplate" + +# shellcheck disable=SC2034 +ubu2004_boilerplate=" +$ubu1910_boilerplate +echo 'Set disable_coredump false' >> /etc/sudo.conf +" + +# shellcheck disable=SC2034 +archlinux_boilerplate=" +pacman -Syu --noconfirm +pacman -S --noconfirm inetutils git curl wget sudo +echo 'Set disable_coredump false' >> /etc/sudo.conf +" + +# shellcheck disable=SC2034 +fedora31_boilerplate=" +dnf update -y +dnf install -y git curl wget hostname +echo 'Set disable_coredump false' >> /etc/sudo.conf +" + +REMOTE_IMAGES=() +CONTAINERS=() +LOCAL_IMAGES=() + +for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do + REMOTE_IMAGES=("${REMOTE_IMAGES[@]}" "${LXC_SUITE[i]}") + CONTAINERS=("${CONTAINERS[@]}" "${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}") + LOCAL_IMAGES=("${LOCAL_IMAGES[@]}" "${LXC_SUITE[i+1]}") +done + +HOST_USER="${SUDO_USER:-$USER}" +HOST_USER_ID=$(id -u "${HOST_USER}") +HOST_GROUP_ID=$(id -g "${HOST_USER}") + +# ---------------------------------------------------------------------------- +usage() { +# ---------------------------------------------------------------------------- + _cmd="$(basename "$0")" + cat <<EOF +usage:: + $_cmd build [containers|<name>] + $_cmd copy [images] + $_cmd remove [containers|<name>|images] + $_cmd [start|stop] [containers|<name>] + $_cmd show [images|suite|info|config [<name>]] + $_cmd cmd [--|<name>] '...' + $_cmd install [suite|base [<name>]] + +build + :containers: build, launch all containers and 'install base' packages + :<name>: build, launch container <name> and 'install base' packages +copy: + :images: copy remote images of the suite into local storage +remove + :containers: delete all 'containers' or only <container-name> + :images: delete local images of the suite +start/stop + :containers: start/stop all 'containers' from the suite + :<name>: start/stop container <name> from suite +show + :info: show info of all (or <name>) containers from LXC suite + :config: show config of all (or <name>) containers from the LXC suite + :suite: show services of all (or <name>) containers from the LXC suite + :images: show information of local images +cmd + use single qoutes to evaluate in container's bash, e.g.: 'echo \$(hostname)' + -- run command '...' in all containers of the LXC suite + :<name>: run command '...' in container <name> +install + :base: prepare LXC; install basic packages + :suite: install LXC ${LXC_SUITE_NAME} suite into all (or <name>) containers + +EOF + usage_containers + [ -n "${1+x}" ] && err_msg "$1" +} + +usage_containers() { + lxc_suite_install_info + [ -n "${1+x}" ] && err_msg "$1" +} + +lxd_info() { + + cat <<EOF + +LXD is needed, to install run:: + + snap install lxd + lxd init --auto + +EOF +} + +main() { + + local exit_val + local _usage="unknown or missing $1 command $2" + + # don't check prerequisite when in recursion + if [[ ! $1 == __* ]]; then + if ! in_container; then + ! required_commands lxc && lxd_info && exit 42 + fi + [[ -z $LXC_SUITE ]] && err_msg "missing LXC_SUITE" && exit 42 + fi + + case $1 in + --getenv) var="$2"; echo "${!var}"; exit 0;; + -h|--help) usage; exit 0;; + + build) + sudo_or_exit + case $2 in + ${LXC_HOST_PREFIX}-*) build_container "$2" ;; + ''|--|containers) build_all_containers ;; + *) usage "$_usage"; exit 42;; + esac + ;; + copy) + case $2 in + ''|images) lxc_copy_images_localy;; + *) usage "$_usage"; exit 42;; + esac + ;; + remove) + sudo_or_exit + case $2 in + ''|--|containers) remove_containers ;; + images) lxc_delete_images_localy ;; + ${LXC_HOST_PREFIX}-*) + ! lxc_exists "$2" && warn_msg "container not yet exists: $2" && exit 0 + if ask_yn "Do you really want to delete container $2"; then + lxc_delete_container "$2" + fi + ;; + *) usage "uknown or missing container <name> $2"; exit 42;; + esac + ;; + start|stop) + sudo_or_exit + case $2 in + ''|--|containers) lxc_cmd "$1" ;; + ${LXC_HOST_PREFIX}-*) + ! lxc_exists "$2" && usage_containers "unknown container: $2" && exit 42 + info_msg "lxc $1 $2" + lxc "$1" "$2" | prefix_stdout "[${_BBlue}${i}${_creset}] " + ;; + *) usage "uknown or missing container <name> $2"; exit 42;; + esac + ;; + show) + sudo_or_exit + case $2 in + suite) + case $3 in + ${LXC_HOST_PREFIX}-*) + lxc exec -t "$3" -- "${LXC_REPO_ROOT}/utils/lxc.sh" __show suite \ + | prefix_stdout "[${_BBlue}$3${_creset}] " + ;; + *) show_suite;; + esac + ;; + images) show_images ;; + config) + case $3 in + ${LXC_HOST_PREFIX}-*) + ! lxc_exists "$3" && usage_containers "unknown container: $3" && exit 42 + lxc config show "$3" | prefix_stdout "[${_BBlue}${3}${_creset}] " + ;; + *) + rst_title "container configurations" + echo + lxc list "$LXC_HOST_PREFIX-" + echo + lxc_cmd config show + ;; + esac + ;; + info) + case $3 in + ${LXC_HOST_PREFIX}-*) + ! lxc_exists "$3" && usage_containers "unknown container: $3" && exit 42 + lxc info "$3" | prefix_stdout "[${_BBlue}${3}${_creset}] " + ;; + *) + rst_title "container info" + echo + lxc_cmd info + ;; + esac + ;; + *) usage "$_usage"; exit 42;; + esac + ;; + __show) + # wrapped show commands, called once in each container + case $2 in + suite) lxc_suite_info ;; + esac + ;; + cmd) + sudo_or_exit + shift + case $1 in + --) shift; lxc_exec "$@" ;; + ${LXC_HOST_PREFIX}-*) + ! lxc_exists "$1" && usage_containers "unknown container: $1" && exit 42 + local name=$1 + shift + lxc_exec_cmd "${name}" "$@" + ;; + *) usage_containers "unknown container: $1" && exit 42 + esac + ;; + install) + sudo_or_exit + case $2 in + suite|base) + case $3 in + ${LXC_HOST_PREFIX}-*) + ! lxc_exists "$3" && usage_containers "unknown container: $3" && exit 42 + lxc_exec_cmd "$3" "${LXC_REPO_ROOT}/utils/lxc.sh" __install "$2" + ;; + ''|--) lxc_exec "${LXC_REPO_ROOT}/utils/lxc.sh" __install "$2" ;; + *) usage_containers "unknown container: $3" && exit 42 + esac + ;; + *) usage "$_usage"; exit 42 ;; + esac + ;; + __install) + # wrapped install commands, called once in each container + # shellcheck disable=SC2119 + case $2 in + suite) lxc_suite_install ;; + base) FORCE_TIMEOUT=0 lxc_install_base_packages ;; + esac + ;; + doc) + echo + echo ".. generic utils/lxc.sh documentation" + ;; + -*) usage "unknown option $1"; exit 42;; + *) usage "unknown or missing command $1"; exit 42;; + esac +} + + +build_all_containers() { + rst_title "Build all LXC containers of suite" + echo + usage_containers + lxc_copy_images_localy + lxc_init_all_containers + lxc_config_all_containers + lxc_boilerplate_all_containers + rst_title "install LXC base packages" section + echo + lxc_exec "${LXC_REPO_ROOT}/utils/lxc.sh" __install base + echo + lxc list "$LXC_HOST_PREFIX" +} + +build_container() { + rst_title "Build container $1" + + local remote_image + local container + local image + local boilerplate_script + + for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do + if [ "${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}" = "$1" ]; then + remote_image="${LXC_SUITE[i]}" + container="${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}" + image="${LXC_SUITE[i+1]}" + boilerplate_script="${image}_boilerplate" + boilerplate_script="${!boilerplate_script}" + break + fi + done + echo + if [ -z "$container" ]; then + err_msg "container $1 unknown" + usage_containers + return 42 + fi + lxc_image_copy "${remote_image}" "${image}" + rst_title "init container" section + lxc_init_container "${image}" "${container}" + rst_title "configure container" section + lxc_config_container "${container}" + rst_title "run LXC boilerplate scripts" section + lxc_install_boilerplate "${container}" "$boilerplate_script" + echo + rst_title "install LXC base packages" section + lxc_exec_cmd "${container}" "${LXC_REPO_ROOT}/utils/lxc.sh" __install base \ + | prefix_stdout "[${_BBlue}${container}${_creset}] " + echo + lxc list "$container" +} + +remove_containers() { + rst_title "Remove all LXC containers of suite" + rst_para "existing containers matching ${_BGreen}$LXC_HOST_PREFIX-*${_creset}" + echo + lxc list "$LXC_HOST_PREFIX-" + echo -en "\\n${_BRed}LXC containers to delete::${_creset}\\n\\n ${CONTAINERS[*]}\\n" | $FMT + local default=Ny + [[ $FORCE_TIMEOUT = 0 ]] && default=Yn + if ask_yn "Do you really want to delete these containers" $default; then + for i in "${CONTAINERS[@]}"; do + lxc_delete_container "$i" + done + fi + echo + lxc list "$LXC_HOST_PREFIX-" +} + +# images +# ------ + +lxc_copy_images_localy() { + rst_title "copy images" section + for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do + lxc_image_copy "${LXC_SUITE[i]}" "${LXC_SUITE[i+1]}" + done + # lxc image list local: && wait_key +} + +lxc_delete_images_localy() { + rst_title "Delete LXC images" + rst_para "local existing images" + echo + lxc image list local: + echo -en "\\n${_BRed}LXC images to delete::${_creset}\\n\\n ${LOCAL_IMAGES[*]}\\n" + if ask_yn "Do you really want to delete these images"; then + for i in "${LOCAL_IMAGES[@]}"; do + lxc_delete_local_image "$i" + done + fi + + for i in $(lxc image list --format csv | grep '^,' | sed 's/,\([^,]*\).*$/\1/'); do + if ask_yn "Image $i has no alias, do you want to delete the image?" Yn; then + lxc_delete_local_image "$i" + fi + done + + echo + lxc image list local: +} + +show_images(){ + rst_title "local images" + echo + lxc image list local: + echo -en "\\n${_Green}LXC suite images::${_creset}\\n\\n ${LOCAL_IMAGES[*]}\\n" + wait_key + for i in "${LOCAL_IMAGES[@]}"; do + if lxc_image_exists "$i"; then + info_msg "lxc image info ${_BBlue}${i}${_creset}" + lxc image info "$i" | prefix_stdout "[${_BBlue}${i}${_creset}] " + else + warn_msg "image ${_BBlue}$i${_creset} does not yet exists" + fi + done + +} + + +# container +# --------- + +show_suite(){ + rst_title "LXC suite ($LXC_HOST_PREFIX-*)" + echo + lxc list "$LXC_HOST_PREFIX-" + echo + for i in "${CONTAINERS[@]}"; do + if ! lxc_exists "$i"; then + warn_msg "container ${_BBlue}$i${_creset} does not yet exists" + else + lxc exec -t "${i}" -- "${LXC_REPO_ROOT}/utils/lxc.sh" __show suite \ + | prefix_stdout "[${_BBlue}${i}${_creset}] " + echo + fi + done +} + +lxc_cmd() { + for i in "${CONTAINERS[@]}"; do + if ! lxc_exists "$i"; then + warn_msg "container ${_BBlue}$i${_creset} does not yet exists" + else + info_msg "lxc $* $i" + lxc "$@" "$i" | prefix_stdout "[${_BBlue}${i}${_creset}] " + fi + done +} + +lxc_exec_cmd() { + local name="$1" + shift + exit_val= + info_msg "[${_BBlue}${name}${_creset}] ${_BGreen}${*}${_creset}" + lxc exec -t --cwd "${LXC_REPO_ROOT}" "${name}" -- bash -c "$*" + exit_val=$? + if [[ $exit_val -ne 0 ]]; then + warn_msg "[${_BBlue}${name}${_creset}] exit code (${_BRed}${exit_val}${_creset}) from ${_BGreen}${*}${_creset}" + else + info_msg "[${_BBlue}${name}${_creset}] exit code (${exit_val}) from ${_BGreen}${*}${_creset}" + fi +} + +lxc_exec() { + for i in "${CONTAINERS[@]}"; do + if ! lxc_exists "$i"; then + warn_msg "container ${_BBlue}$i${_creset} does not yet exists" + else + lxc_exec_cmd "${i}" "$@" | prefix_stdout "[${_BBlue}${i}${_creset}] " + fi + done +} + +lxc_init_all_containers() { + rst_title "init all containers" section + + local image_name + local container_name + + for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do + lxc_init_container "${LXC_SUITE[i+1]}" "${LXC_HOST_PREFIX}-${LXC_SUITE[i+1]}" + done +} + +lxc_config_all_containers() { + rst_title "configure all containers" section + + for i in "${CONTAINERS[@]}"; do + lxc_config_container "${i}" + done +} + +lxc_config_container() { + info_msg "[${_BBlue}$1${_creset}] configure container ..." + + info_msg "[${_BBlue}$1${_creset}] map uid/gid from host to container" + # https://lxd.readthedocs.io/en/latest/userns-idmap/#custom-idmaps + echo -e -n "uid $HOST_USER_ID 0\\ngid $HOST_GROUP_ID 0"\ + | lxc config set "$1" raw.idmap - + + info_msg "[${_BBlue}$1${_creset}] share ${REPO_ROOT} (repo_share) from HOST into container" + # https://lxd.readthedocs.io/en/latest/instances/#type-disk + lxc config device add "$1" repo_share disk \ + source="${REPO_ROOT}" \ + path="${LXC_REPO_ROOT}" &>/dev/null + # lxc config show "$1" && wait_key +} + +lxc_boilerplate_all_containers() { + rst_title "run LXC boilerplate scripts" section + + local boilerplate_script + local image_name + + for ((i=0; i<${#LXC_SUITE[@]}; i+=2)); do + + image_name="${LXC_SUITE[i+1]}" + boilerplate_script="${image_name}_boilerplate" + boilerplate_script="${!boilerplate_script}" + + lxc_install_boilerplate "${LXC_HOST_PREFIX}-${image_name}" "$boilerplate_script" + + if [[ -z "${boilerplate_script}" ]]; then + err_msg "[${_BBlue}${container_name}${_creset}] no boilerplate for image '${image_name}'" + fi + done +} + +lxc_install_boilerplate() { + + # usage: lxc_install_boilerplate <container-name> <string: shell commands ..> + # + # usage: lxc_install_boilerplate searx-archlinux "${archlinux_boilerplate}" + + local container_name="$1" + local boilerplate_script="$2" + + info_msg "[${_BBlue}${container_name}${_creset}] init .." + if lxc start -q "${container_name}" &>/dev/null; then + sleep 5 # guest needs some time to come up and get an IP + fi + lxc_init_container_env "${container_name}" + info_msg "[${_BBlue}${container_name}${_creset}] install /.lxcenv.mk .." + cat <<EOF | lxc exec "${container_name}" -- bash | prefix_stdout "[${_BBlue}${container_name}${_creset}] " +rm -f "/.lxcenv.mk" +ln -s "${LXC_REPO_ROOT}/utils/makefile.lxc" "/.lxcenv.mk" +ls -l "/.lxcenv.mk" +EOF + + info_msg "[${_BBlue}${container_name}${_creset}] run LXC boilerplate scripts .." + if lxc start -q "${container_name}" &>/dev/null; then + sleep 5 # guest needs some time to come up and get an IP + fi + if [[ -n "${boilerplate_script}" ]]; then + echo "${boilerplate_script}" \ + | lxc exec "${container_name}" -- bash \ + | prefix_stdout "[${_BBlue}${container_name}${_creset}] " + fi +} + + +# ---------------------------------------------------------------------------- +main "$@" +# ---------------------------------------------------------------------------- diff --git a/utils/makefile.include b/utils/makefile.include index 716889c02..65aca70f0 100644 --- a/utils/makefile.include +++ b/utils/makefile.include @@ -1,12 +1,25 @@ # -*- coding: utf-8; mode: makefile-gmake -*- +ifeq (,$(wildcard /.lxcenv.mk)) +PHONY += lxc-activate lxc-purge +lxc-activate: + @$(MAKE) -s -f /share/searx/utils/makefile.lxc lxc-activate +lxc-purge: + $(Q)rm -rf ./lxc +else + include /.lxcenv.mk +endif + +ifeq (,$(wildcard /.lxcenv.mk)) make-help: +else +make-help: lxc-help +endif @echo ' make V=0|1 [targets] 0 => quiet build (default), 1 => verbose build' @echo ' make V=2 [targets] 2 => give reason for rebuild of target' quiet_cmd_common_clean = CLEAN $@ cmd_common_clean = \ - rm -rf tests/build ;\ find . -name '*.orig' -exec rm -f {} + ;\ find . -name '*.rej' -exec rm -f {} + ;\ find . -name '*~' -exec rm -f {} + ;\ @@ -126,3 +139,4 @@ echo-cmd = $(if $($(quiet)cmd_$(1)),echo '$(call escsq,$($(quiet)cmd_$(1)))$(ech # printing commands cmd = @$(echo-cmd) $(cmd_$(1)) +.PHONY: $(PHONY) diff --git a/utils/makefile.lxc b/utils/makefile.lxc new file mode 100644 index 000000000..fd999e40d --- /dev/null +++ b/utils/makefile.lxc @@ -0,0 +1,29 @@ +# -*- coding: utf-8; mode: makefile-gmake -*- +# +# LXC environment +# =============== +# +# To activate/deactivate LXC makefile environment in a container, set/unset link +# from root '/.lxcenv.mk' to *this* file:: +# +# sudo make ./utils/makefile.lxc lxc-activate +# sudo make ./utils/makefile.lxc lxc-deactivate + +LXC_ENV_FOLDER=lxc/$(shell hostname)/ + +lxc-help:: + @echo 'LXC: running in container LXC_ENV_FOLDER=$(LXC_ENV_FOLDER)' + +# If not activated, serve target 'lxc-activate' .. +ifeq (,$(wildcard /.lxcenv.mk)) +PHONY += lxc-activate +lxc-activate: + ln -s "$(abspath $(lastword $(MAKEFILE_LIST)))" "/.lxcenv.mk" +else +# .. and if activated, serve target 'lxc-deactivate'. +PHONY += lxc-deactivate +lxc-deactivate: + rm /.lxcenv.mk +endif + +.PHONY: $(PHONY) diff --git a/utils/makefile.python b/utils/makefile.python index 590bbdb46..df16acbbf 100644 --- a/utils/makefile.python +++ b/utils/makefile.python @@ -8,9 +8,9 @@ export PYTHONPATH := $(SITE_PYTHON):$$PYTHONPATH export PY_ENV PYDIST PYBUILD # folder where the python distribution takes place -PYDIST ?= ./py_dist +PYDIST = ./$(LXC_ENV_FOLDER)dist # folder where the python intermediate build files take place -PYBUILD ?= ./py_build +PYBUILD = ./$(LXC_ENV_FOLDER)build # python version to use PY ?=3 # $(PYTHON) points to the python interpreter from the OS! The python from the @@ -30,8 +30,7 @@ PYLINT_RC ?= .pylintrc TEST_FOLDER ?= ./tests TEST ?= . -VTENV_OPTS = "--no-site-packages" -PY_ENV = ./local/py$(PY) +PY_ENV = ./$(LXC_ENV_FOLDER)local/py$(PY) PY_ENV_BIN = $(PY_ENV)/bin PY_ENV_ACT = . $(PY_ENV_BIN)/activate @@ -41,6 +40,7 @@ ifeq ($(OS),Windows_NT) PY_ENV_ACT = $(PY_ENV_BIN)/activate endif +VTENV_OPTS ?= ifeq ($(PYTHON),python) VIRTUALENV = virtualenv else diff --git a/utils/makefile.sphinx b/utils/makefile.sphinx index 2c1922fc9..1b0f42ccb 100644 --- a/utils/makefile.sphinx +++ b/utils/makefile.sphinx @@ -1,17 +1,19 @@ # -*- coding: utf-8; mode: makefile-gmake -*- +export DOCS_FOLDER DOCS_BUILD DOCS_DIST BOOKS_FOLDER BOOKS_DIST + # You can set these variables from the command line. SPHINXOPTS ?= SPHINXBUILD ?= $(PY_ENV_BIN)/sphinx-build SPHINX_CONF ?= conf.py -DOCS_FOLDER ?= docs -DOCS_BUILD ?= build/docs -DOCS_DIST ?= dist/docs +DOCS_FOLDER = ./docs +DOCS_BUILD = ./$(LXC_ENV_FOLDER)build/docs +DOCS_DIST = ./$(LXC_ENV_FOLDER)dist/docs GH_PAGES ?= gh-pages -BOOKS_FOLDER ?= docs -BOOKS_DIST ?= dist/books +BOOKS_FOLDER = ./docs +BOOKS_DIST = ./$(LXC_ENV_FOLDER)dist/books ifeq ($(KBUILD_VERBOSE),1) SPHINX_VERBOSE = "-v" @@ -54,11 +56,13 @@ docs-help: # requirements # ------------------------------------------------------------------------------ -sphinx-doc: $(PY_ENV) +sphinx-doc-prebuilds:: $(PY_ENV) + +sphinx-doc: sphinx-doc-prebuilds @echo "PYENV installing Sphinx$(SPHINXVERS)" $(Q)$(PY_ENV_BIN)/pip install $(PIP_VERBOSE) 'Sphinx$(SPHINXVERS)' -sphinx-live: $(PY_ENV) +sphinx-live: sphinx-doc-prebuilds @echo "PYENV installing Sphinx$(SPHINXVERS)" $(Q)$(PY_ENV_BIN)/pip install $(PIP_VERBOSE) 'Sphinx$(SPHINXVERS)' sphinx-autobuild @@ -113,7 +117,7 @@ quiet_cmd_sphinx_clean = CLEAN $@ # targets # ------------------------------------------------------------------------------ -# build PDF of whole documentation in: $(DOCS_DIST)/pdf +# build PDF of whole documentation in: $(DOCS_DIST)/pdf PHONY += sphinx-pdf sphinx-pdf: sphinx-latex @@ -154,7 +158,7 @@ $(BOOKS_HTML): sphinx-doc | $(BOOKS_DIST) -b html \ -c $(DOCS_FOLDER) \ -d $(DOCS_BUILD)/books/$(patsubst books/%.html,%,$@)/.doctrees \ - $(patsubst books/%.html,%,$@) \ + $(BOOKS_FOLDER)/$(patsubst books/%.html,%,$@) \ $(BOOKS_DIST)/$(patsubst books/%.html,%,$@) @echo "SPHINX $@ --> file://$(abspath $(BOOKS_DIST)/$(patsubst books/%.html,%,$@))" @@ -166,7 +170,7 @@ $(BOOKS_LIVE): sphinx-live | $(BOOKS_DIST) -b html \ -c $(DOCS_FOLDER) \ -d $(DOCS_BUILD)/books/$(patsubst books/%.live,%,$@)/.doctrees \ - $(patsubst books/%.live,%,$@) \ + $(BOOKS_FOLDER)/$(patsubst books/%.live,%,$@) \ $(BOOKS_DIST)/$(patsubst books/%.live,%,$@) $(BOOKS_PDF): %.pdf : %.latex @@ -182,7 +186,7 @@ $(BOOKS_LATEX): sphinx-doc | $(BOOKS_DIST) -b latex \ -c $(DOCS_FOLDER) \ -d $(DOCS_BUILD)/books/$(patsubst books/%.latex,%,$@)/.doctrees \ - $(patsubst books/%.latex,%,$@) \ + $(BOOKS_FOLDER)/$(patsubst books/%.latex,%,$@) \ $(DOCS_BUILD)/latex/$(patsubst books/%.latex,%,$@) @echo "SPHINX $@ --> file://$(abspath $(DOCS_BUILD)/latex/$(patsubst books/%.latex,%,$@))" diff --git a/utils/morty.sh b/utils/morty.sh new file mode 100755 index 000000000..75bfeeedf --- /dev/null +++ b/utils/morty.sh @@ -0,0 +1,546 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- +# SPDX-License-Identifier: AGPL-3.0-or-later + +# shellcheck source=utils/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +# shellcheck source=utils/brand.env +source "${REPO_ROOT}/utils/brand.env" +source_dot_config +SEARX_URL="${PUBLIC_URL:-http://$(uname -n)/searx}" +source "${REPO_ROOT}/utils/lxc-searx.env" +in_container && lxc_set_suite_env + +# ---------------------------------------------------------------------------- +# config +# ---------------------------------------------------------------------------- + +MORTY_LISTEN="${MORTY_LISTEN:-127.0.0.1:3000}" +PUBLIC_URL_PATH_MORTY="${PUBLIC_URL_PATH_MORTY:-/morty/}" + +PUBLIC_URL_MORTY="${PUBLIC_URL_MORTY:-$(echo "$SEARX_URL" | sed -e's,^\(.*://[^/]*\).*,\1,g')${PUBLIC_URL_PATH_MORTY}}" + +# shellcheck disable=SC2034 +MORTY_TIMEOUT=5 + +SERVICE_NAME="morty" +SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}" +SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}" +SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}" +SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service" +# shellcheck disable=SC2034 +SERVICE_GROUP="${SERVICE_USER}" +# shellcheck disable=SC2034 +SERVICE_ENV_DEBUG=false + +GO_ENV="${SERVICE_HOME}/.go_env" +GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz" +GO_TAR=$(basename "$GO_PKG_URL") + +# shellcheck disable=SC2034 +CONFIG_FILES=() + +# Apache Settings + +APACHE_MORTY_SITE="morty.conf" +NGINX_MORTY_SITE="morty.conf" + +# ---------------------------------------------------------------------------- +usage() { +# ---------------------------------------------------------------------------- + + # shellcheck disable=SC1117 + cat <<EOF +usage:: + $(basename "$0") shell + $(basename "$0") install [all|user] + $(basename "$0") update [morty] + $(basename "$0") remove [all] + $(basename "$0") activate [service] + $(basename "$0") deactivate [service] + $(basename "$0") inspect [service] + $(basename "$0") option [debug-on|debug-off|new-key] + $(basename "$0") apache [install|remove] + $(basename "$0") nginx [install|remove] + $(basename "$0") info [searx] + +shell + start interactive shell from user ${SERVICE_USER} +install / remove + all: complete setup of morty service + user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME) +update morty + Update morty installation ($SERVICE_HOME) +activate service + activate and start service daemon (systemd unit) +deactivate service + stop and deactivate service daemon (systemd unit) +inspect service + show service status and log +option + set one of the available options + :new-key: set new morty key +apache : ${PUBLIC_URL_MORTY} + :install: apache site with a reverse proxy (ProxyPass) + :remove: apache site ${APACHE_MORTY_SITE} +nginx (${PUBLIC_URL_MORTY}) + :install: nginx site with a reverse proxy (ProxyPass) + :remove: nginx site ${NGINX_MORTY_SITE} + +If needed, set the environment variables in the '${DOT_CONFIG#"$REPO_ROOT/"}' file:: + PUBLIC_URL_MORTY: ${PUBLIC_URL_MORTY} + MORTY_LISTEN: ${MORTY_LISTEN} + SERVICE_USER: ${SERVICE_USER} +EOF + if in_container; then + # in containers the service is listening on 0.0.0.0 (see lxc-searx.env) + for ip in $(global_IPs) ; do + if [[ $ip =~ .*:.* ]]; then + echo " container URL (IPv6): http://[${ip#*|}]:3000/" + else + # IPv4: + echo " container URL (IPv4): http://${ip#*|}:3000/" + fi + done + fi + echo + info_searx + + [[ -n ${1} ]] && err_msg "$1" +} + +info_searx() { + # shellcheck disable=SC1117 + cat <<EOF +To activate result and image proxy in searx, edit settings.yml (read: +${DOCS_URL}/admin/morty.html):: + result_proxy: + url : ${PUBLIC_URL_MORTY} + server: + image_proxy : True +EOF +} + +main() { + required_commands \ + sudo install git wget curl \ + || exit + + local _usage="ERROR: unknown or missing $1 command $2" + + case $1 in + --getenv) var="$2"; echo "${!var}"; exit 0;; + -h|--help) usage; exit 0;; + + shell) + sudo_or_exit + interactive_shell "${SERVICE_USER}" + ;; + inspect) + case $2 in + service) + sudo_or_exit + inspect_service + ;; + *) usage "$_usage"; exit 42;; + esac ;; + install) + rst_title "$SERVICE_NAME" part + sudo_or_exit + case $2 in + all) install_all ;; + user) assert_user ;; + *) usage "$_usage"; exit 42;; + esac ;; + update) + sudo_or_exit + case $2 in + morty) update_morty ;; + *) usage "$_usage"; exit 42;; + esac ;; + remove) + sudo_or_exit + case $2 in + all) remove_all;; + user) drop_service_account "${SERVICE_USER}" ;; + *) usage "$_usage"; exit 42;; + esac ;; + activate) + sudo_or_exit + case $2 in + service) systemd_activate_service "${SERVICE_NAME}" ;; + *) usage "$_usage"; exit 42;; + esac ;; + deactivate) + sudo_or_exit + case $2 in + service) systemd_deactivate_service "${SERVICE_NAME}" ;; + *) usage "$_usage"; exit 42;; + esac ;; + apache) + sudo_or_exit + case $2 in + install) install_apache_site ;; + remove) remove_apache_site ;; + *) usage "$_usage"; exit 42;; + esac ;; + nginx) + sudo_or_exit + case $2 in + install) install_nginx_site ;; + remove) remove_nginx_site ;; + *) usage "$_usage"; exit 42;; + esac ;; + info) + case $2 in + searx) info_searx ;; + *) usage "$_usage"; exit 42;; + esac ;; + option) + sudo_or_exit + case $2 in + new-key) set_new_key ;; + debug-on) enable_debug ;; + debug-off) disable_debug ;; + *) usage "$_usage"; exit 42;; + esac ;; + doc) rst-doc ;; + *) usage "ERROR: unknown or missing command $1"; exit 42;; + esac +} + +install_all() { + + MORTY_KEY="$(head -c 32 /dev/urandom | base64)" + + rst_title "Install $SERVICE_NAME (service)" + assert_user + wait_key + install_go "${GO_PKG_URL}" "${GO_TAR}" "${SERVICE_USER}" + wait_key + install_morty + wait_key + systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" + wait_key + if ! service_is_available "http://${MORTY_LISTEN}" ; then + err_msg "Morty does not listening on: http://${MORTY_LISTEN}" + fi + if apache_is_installed; then + info_msg "Apache is installed on this host." + if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then + install_apache_site + fi + elif nginx_is_installed; then + info_msg "nginx is installed on this host." + if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then + install_nginx_site + fi + fi + info_searx + if ask_yn "Add image and result proxy to searx settings.yml?" Yn; then + "${REPO_ROOT}/utils/searx.sh" option result-proxy "${PUBLIC_URL_MORTY}" "${MORTY_KEY}" + "${REPO_ROOT}/utils/searx.sh" option image-proxy-on + fi + + if ask_yn "Do you want to inspect the installation?" Ny; then + inspect_service + fi + +} + +remove_all() { + rst_title "De-Install $SERVICE_NAME (service)" + + rst_para "\ +It goes without saying that this script can only be used to remove +installations that were installed with this script." + + if systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then + drop_service_account "${SERVICE_USER}" + fi +} + +assert_user() { + rst_title "user $SERVICE_USER" section + echo + tee_stderr 1 <<EOF | bash | prefix_stdout +useradd --shell /bin/bash --system \ + --home-dir "$SERVICE_HOME" \ + --comment 'Web content sanitizer proxy' $SERVICE_USER +mkdir "$SERVICE_HOME" +chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME" +groups $SERVICE_USER +EOF + SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)" + export SERVICE_HOME + echo "export SERVICE_HOME=$SERVICE_HOME" + + cat > "$GO_ENV" <<EOF +export GOPATH=\$HOME/go-apps +export PATH=\$PATH:\$HOME/local/go/bin:\$GOPATH/bin +EOF + echo "Environment $GO_ENV has been setup." + + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" +grep -qFs -- 'source $GO_ENV' ~/.profile || echo 'source $GO_ENV' >> ~/.profile +EOF +} + +morty_is_installed() { + [[ -f $SERVICE_HOME/go-apps/bin/morty ]] +} + +_svcpr=" ${_Yellow}|${SERVICE_USER}|${_creset} " + +install_morty() { + rst_title "Install morty in user's ~/go-apps" section + echo + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr" +go get -v -u github.com/asciimoo/morty +EOF + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr" +cd \$GOPATH/src/github.com/asciimoo/morty +go test +go test -benchmem -bench . +EOF +} + +update_morty() { + rst_title "Update morty" section + echo + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr" +go get -v -u github.com/asciimoo/morty +EOF + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_svcpr" +cd \$GOPATH/src/github.com/asciimoo/morty +go test +go test -benchmem -bench . +EOF +} + +set_service_env_debug() { + + # usage: set_service_env_debug [false|true] + + # shellcheck disable=SC2034 + local SERVICE_ENV_DEBUG="${1:-false}" + if systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then + systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" + fi +} + +inspect_service() { + + rst_title "service status & log" + + cat <<EOF + +sourced ${DOT_CONFIG#"$REPO_ROOT/"} : + + MORTY_LISTEN : ${MORTY_LISTEN} + +EOF + + if service_account_is_available "$SERVICE_USER"; then + info_msg "service account $SERVICE_USER available." + else + err_msg "service account $SERVICE_USER not available!" + fi + if go_is_available "$SERVICE_USER"; then + info_msg "~$SERVICE_USER: go is installed" + else + err_msg "~$SERVICE_USER: go is not installed" + fi + if morty_is_installed; then + info_msg "~$SERVICE_USER: morty app is installed" + else + err_msg "~$SERVICE_USER: morty app is not installed!" + fi + + if ! service_is_available "http://${MORTY_LISTEN}" ; then + err_msg "Morty does not listening on: http://${MORTY_LISTEN}" + echo -e "${_Green}stop with [${_BCyan}CTRL-C${_Green}] or .." + wait_key + fi + + if ! service_is_available "${PUBLIC_URL_MORTY}"; then + warn_msg "Public service at ${PUBLIC_URL_MORTY} is not available!" + if ! in_container; then + warn_msg "Check if public name is correct and routed or use the public IP from above." + fi + fi + + if in_container; then + lxc_suite_info + else + info_msg "public URL --> ${PUBLIC_URL_MORTY}" + info_msg "morty URL --> http://${MORTY_LISTEN}" + fi + + local _debug_on + if ask_yn "Enable morty debug mode (needs reinstall of systemd service)?"; then + enable_debug + _debug_on=1 + else + systemctl --no-pager -l status "${SERVICE_NAME}" + fi + echo + + # shellcheck disable=SC2059 + printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log" + read -r -s -n1 -t 5 + echo + while true; do + trap break 2 + journalctl -f -u "${SERVICE_NAME}" + done + + if [[ $_debug_on == 1 ]]; then + FORCE_SELECTION=Y disable_debug + fi + return 0 +} + +enable_debug() { + warn_msg "Do not enable debug in production enviroments!!" + info_msg "Enabling debug option needs to reinstall systemd service!" + set_service_env_debug true +} + +disable_debug() { + info_msg "Disabling debug option needs to reinstall systemd service!" + set_service_env_debug false +} + + +set_new_key() { + rst_title "Set morty key" + echo + + MORTY_KEY="$(head -c 32 /dev/urandom | base64)" + info_msg "morty key: '${MORTY_KEY}'" + + warn_msg "this will need to reinstall services .." + MSG="${_Green}press any [${_BCyan}KEY${_Green}] to continue // stop with [${_BCyan}CTRL-C${_creset}]" wait_key + + systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}" + "${REPO_ROOT}/utils/searx.sh" option result-proxy "${PUBLIC_URL_MORTY}" "${MORTY_KEY}" + "${REPO_ROOT}/utils/searx.sh" option image-proxy-on +} + + +install_apache_site() { + + rst_title "Install Apache site $APACHE_MORTY_SITE" + + rst_para "\ +This installs a reverse proxy (ProxyPass) into apache site (${APACHE_MORTY_SITE})" + + ! apache_is_installed && err_msg "Apache is not installed." + + if ! ask_yn "Do you really want to continue?" Yn; then + return + else + install_apache + fi + + apache_install_site "${APACHE_MORTY_SITE}" + + info_msg "testing public url .." + if ! service_is_available "${PUBLIC_URL_MORTY}"; then + err_msg "Public service at ${PUBLIC_URL_MORTY} is not available!" + fi +} + +remove_apache_site() { + + rst_title "Remove Apache site $APACHE_MORTY_SITE" + + rst_para "\ +This removes apache site ${APACHE_MORTY_SITE}." + + ! apache_is_installed && err_msg "Apache is not installed." + + if ! ask_yn "Do you really want to continue?" Yn; then + return + fi + + apache_remove_site "$APACHE_MORTY_SITE" +} + +install_nginx_site() { + + rst_title "Install nginx site $NGINX_MORTY_SITE" + + rst_para "\ +This installs a reverse proxy (ProxyPass) into nginx site (${NGINX_MORTY_SITE})" + + ! nginx_is_installed && err_msg "nginx is not installed." + + if ! ask_yn "Do you really want to continue?" Yn; then + return + else + install_nginx + fi + + "${REPO_ROOT}/utils/searx.sh" install uwsgi + + # shellcheck disable=SC2034 + SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC) + # shellcheck disable=SC2034 + SEARX_URL_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_URL_PATH) + nginx_install_app "${NGINX_MORTY_SITE}" + + info_msg "testing public url .." + if ! service_is_available "${PUBLIC_URL_MORTY}"; then + err_msg "Public service at ${PUBLIC_URL_MORTY} is not available!" + fi +} + +remove_nginx_site() { + + rst_title "Remove nginx site $NGINX_MORTY_SITE" + + rst_para "\ +This removes nginx site ${NGINX_MORTY_SITE}." + + ! nginx_is_installed && err_msg "nginx is not installed." + + if ! ask_yn "Do you really want to continue?" Yn; then + return + fi + + nginx_remove_site "$NGINX_MORTY_SITE" + +} + +rst-doc() { + + eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/morty.rst")\"" + + echo -e "\n.. START install systemd unit" + cat <<EOF +.. tabs:: + + .. group-tab:: systemd + + .. code:: bash + +EOF + eval "echo \"$(< "${TEMPLATES}/${SERVICE_SYSTEMD_UNIT}")\"" | prefix_stdout " " + echo -e "\n.. END install systemd unit" + + # for DIST_NAME in ubuntu-20.04 arch fedora; do + # ( + # DIST_ID=${DIST_NAME%-*} + # DIST_VERS=${DIST_NAME#*-} + # [[ $DIST_VERS =~ $DIST_ID ]] && DIST_VERS= + # # ... + # ) + # done +} + + +# ---------------------------------------------------------------------------- +main "$@" +# ---------------------------------------------------------------------------- diff --git a/utils/searx.sh b/utils/searx.sh new file mode 100755 index 000000000..4009fddb0 --- /dev/null +++ b/utils/searx.sh @@ -0,0 +1,869 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- +# SPDX-License-Identifier: AGPL-3.0-or-later +# shellcheck disable=SC2001 + +# shellcheck source=utils/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +# shellcheck source=utils/brand.env +source "${REPO_ROOT}/utils/brand.env" +source_dot_config +source "${REPO_ROOT}/utils/lxc-searx.env" +in_container && lxc_set_suite_env + +# ---------------------------------------------------------------------------- +# config +# ---------------------------------------------------------------------------- + +PUBLIC_URL="${PUBLIC_URL:-http://$(uname -n)/searx}" + +SEARX_INTERNAL_HTTP="${SEARX_INTERNAL_HTTP:-127.0.0.1:8888}" + +SEARX_URL_PATH="${SEARX_URL_PATH:-$(echo "${PUBLIC_URL}" \ +| sed -e 's,^.*://[^/]*\(/.*\),\1,g')}" +[[ "${SEARX_URL_PATH}" == "${PUBLIC_URL}" ]] && SEARX_URL_PATH=/ +SEARX_INSTANCE_NAME="${SEARX_INSTANCE_NAME:-searx@$(echo "$PUBLIC_URL" \ +| sed -e 's,^.*://\([^\:/]*\).*,\1,g') }" + +SERVICE_NAME="searx" +SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}" +SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}" +SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}" +# shellcheck disable=SC2034 +SERVICE_GROUP="${SERVICE_USER}" + +GIT_BRANCH="${GIT_BRANCH:-master}" +SEARX_PYENV="${SERVICE_HOME}/searx-pyenv" +SEARX_SRC="${SERVICE_HOME}/searx-src" +SEARX_SETTINGS_PATH="/etc/searx/settings.yml" +SEARX_UWSGI_APP="searx.ini" +# shellcheck disable=SC2034 +SEARX_UWSGI_SOCKET="/run/uwsgi/app/searx/socket" + +# apt packages +SEARX_PACKAGES_debian="\ +virtualenv python3-dev python3-babel python3-venv +uwsgi uwsgi-plugin-python3 +git build-essential libxslt-dev zlib1g-dev libffi-dev libssl-dev +shellcheck" + +BUILD_PACKAGES_debian="\ +firefox graphviz imagemagick texlive-xetex librsvg2-bin +texlive-latex-recommended texlive-extra-utils ttf-dejavu +latexmk" + +# pacman packages +SEARX_PACKAGES_arch="\ +python-virtualenv python python-pip python-lxml python-babel +uwsgi uwsgi-plugin-python +git base-devel libxml2 +shellcheck" + +BUILD_PACKAGES_arch="\ +firefox graphviz imagemagick texlive-bin extra/librsvg +texlive-core texlive-latexextra ttf-dejavu" + +# dnf packages +SEARX_PACKAGES_fedora="\ +virtualenv python python-pip python-lxml python-babel +uwsgi uwsgi-plugin-python3 +git @development-tools libxml2 +ShellCheck" + +BUILD_PACKAGES_fedora="\ +firefox graphviz graphviz-gd ImageMagick librsvg2-tools +texlive-xetex-bin texlive-collection-fontsrecommended +texlive-collection-latex dejavu-sans-fonts dejavu-serif-fonts +dejavu-sans-mono-fonts" + +case $DIST_ID-$DIST_VERS in + ubuntu-16.04|ubuntu-18.04) + SEARX_PACKAGES="${SEARX_PACKAGES_debian}" + BUILD_PACKAGES="${BUILD_PACKAGES_debian}" + APACHE_PACKAGES="$APACHE_PACKAGES libapache2-mod-proxy-uwsgi" + ;; + ubuntu-20.04) + # https://askubuntu.com/a/1224710 + SEARX_PACKAGES="${SEARX_PACKAGES_debian} python-is-python3" + BUILD_PACKAGES="${BUILD_PACKAGES_debian}" + ;; + ubuntu-*|debian-*) + SEARX_PACKAGES="${SEARX_PACKAGES_debian}" + BUILD_PACKAGES="${BUILD_PACKAGES_debian}" + ;; + arch-*) + SEARX_PACKAGES="${SEARX_PACKAGES_arch}" + BUILD_PACKAGES="${BUILD_PACKAGES_arch}" + ;; + fedora-*) + SEARX_PACKAGES="${SEARX_PACKAGES_fedora}" + BUILD_PACKAGES="${BUILD_PACKAGES_fedora}" + ;; +esac + +# Apache Settings +APACHE_SEARX_SITE="searx.conf" + +# shellcheck disable=SC2034 +CONFIG_FILES=( + "${uWSGI_APPS_AVAILABLE}/${SEARX_UWSGI_APP}" +) + +# shellcheck disable=SC2034 +CONFIG_BACKUP_ENCRYPTED=( + "${SEARX_SETTINGS_PATH}" +) + +# ---------------------------------------------------------------------------- +usage() { +# ---------------------------------------------------------------------------- + + # shellcheck disable=SC1117 + cat <<EOF +usage:: + $(basename "$0") shell + $(basename "$0") install [all|user|searx-src|pyenv|uwsgi|packages|buildhost] + $(basename "$0") update [searx] + $(basename "$0") remove [all|user|pyenv|searx-src] + $(basename "$0") activate [service] + $(basename "$0") deactivate [service] + $(basename "$0") inspect [service] + $(basename "$0") option [debug-[on|off]|image-proxy-[on|off]|result-proxy <url> <key>] + $(basename "$0") apache [install|remove] + +shell + start interactive shell from user ${SERVICE_USER} +install / remove + :all: complete (de-) installation of searx service + :user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME) + :searx-src: clone $GIT_URL + :pyenv: create/remove virtualenv (python) in $SEARX_PYENV + :uwsgi: install searx uWSGI application + :settings: reinstall settings from ${REPO_ROOT}/searx/settings.yml + :packages: install needed packages from OS package manager + :buildhost: install packages from OS package manager needed by buildhosts +update searx + Update searx installation ($SERVICE_HOME) +activate service + activate and start service daemon (systemd unit) +deactivate service + stop and deactivate service daemon (systemd unit) +inspect service + run some small tests and inspect service's status and log +option + set one of the available options +apache + :install: apache site with the searx uwsgi app + :remove: apache site ${APACHE_FILTRON_SITE} + +searx settings: ${SEARX_SETTINGS_PATH} + +If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/"}' file:: + PUBLIC_URL : ${PUBLIC_URL} + SEARX_INSTANCE_NAME : ${SEARX_INSTANCE_NAME} + SERVICE_USER : ${SERVICE_USER} + SEARX_INTERNAL_HTTP : http://${SEARX_INTERNAL_HTTP} +EOF + if in_container; then + # searx is listening on 127.0.0.1 and not available from outside container + # in containers the service is listening on 0.0.0.0 (see lxc-searx.env) + echo -e "${_BBlack}HINT:${_creset} searx only listen on loopback device" \ + "${_BBlack}inside${_creset} the container." + for ip in $(global_IPs) ; do + if [[ $ip =~ .*:.* ]]; then + echo " container (IPv6): [${ip#*|}]" + else + # IPv4: + echo " container (IPv4): ${ip#*|}" + fi + done + fi + [[ -n ${1} ]] && err_msg "$1" +} + +main() { + required_commands \ + sudo systemctl install git wget curl \ + || exit + + local _usage="unknown or missing $1 command $2" + + case $1 in + --getenv) var="$2"; echo "${!var}"; exit 0;; + -h|--help) usage; exit 0;; + shell) + sudo_or_exit + interactive_shell "${SERVICE_USER}" + ;; + inspect) + case $2 in + service) + sudo_or_exit + inspect_service + ;; + *) usage "$_usage"; exit 42;; + esac ;; + install) + rst_title "$SEARX_INSTANCE_NAME" part + sudo_or_exit + case $2 in + all) install_all ;; + user) assert_user ;; + pyenv) create_pyenv ;; + searx-src) clone_searx ;; + settings) install_settings ;; + uwsgi) + install_searx_uwsgi + if ! service_is_available "http://${SEARX_INTERNAL_HTTP}"; then + err_msg "URL http://${SEARX_INTERNAL_HTTP} not available, check searx & uwsgi setup!" + fi + ;; + packages) + pkg_install "$SEARX_PACKAGES" + ;; + buildhost) + pkg_install "$SEARX_PACKAGES" + pkg_install "$BUILD_PACKAGES" + ;; + *) usage "$_usage"; exit 42;; + esac ;; + update) + sudo_or_exit + case $2 in + searx) update_searx;; + *) usage "$_usage"; exit 42;; + esac ;; + remove) + sudo_or_exit + case $2 in + all) remove_all;; + user) drop_service_account "${SERVICE_USER}";; + pyenv) remove_pyenv ;; + searx-src) remove_searx ;; + *) usage "$_usage"; exit 42;; + esac ;; + activate) + sudo_or_exit + case $2 in + service) + activate_service ;; + *) usage "$_usage"; exit 42;; + esac ;; + deactivate) + sudo_or_exit + case $2 in + service) deactivate_service ;; + *) usage "$_usage"; exit 42;; + esac ;; + option) + sudo_or_exit + case $2 in + debug-on) echo; enable_debug ;; + debug-off) echo; disable_debug ;; + result-proxy) set_result_proxy "$3" "$4" ;; + image-proxy-on) enable_image_proxy ;; + image-proxy-off) disable_image_proxy ;; + *) usage "$_usage"; exit 42;; + esac ;; + apache) + sudo_or_exit + case $2 in + install) install_apache_site ;; + remove) remove_apache_site ;; + *) usage "$_usage"; exit 42;; + esac ;; + doc) rst-doc;; + *) usage "unknown or missing command $1"; exit 42;; + esac +} + +_service_prefix=" ${_Yellow}|$SERVICE_USER|${_creset} " + +install_all() { + rst_title "Install $SEARX_INSTANCE_NAME (service)" + pkg_install "$SEARX_PACKAGES" + wait_key + assert_user + wait_key + clone_searx + wait_key + create_pyenv + wait_key + install_settings + wait_key + test_local_searx + wait_key + install_searx_uwsgi + if ! service_is_available "http://${SEARX_INTERNAL_HTTP}"; then + err_msg "URL http://${SEARX_INTERNAL_HTTP} not available, check searx & uwsgi setup!" + fi + if ask_yn "Do you want to inspect the installation?" Ny; then + inspect_service + fi +} + +update_searx() { + rst_title "Update searx instance" + + echo + tee_stderr 0.3 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +git checkout -B "$GIT_BRANCH" +git pull +pip install -U pip +pip install -U setuptools +pip install -U wheel +pip install -U -e . +EOF + install_settings + uWSGI_restart "$SEARX_UWSGI_APP" +} + +remove_all() { + rst_title "De-Install $SEARX_INSTANCE_NAME (service)" + + rst_para "\ +It goes without saying that this script can only be used to remove +installations that were installed with this script." + + if ! ask_yn "Do you really want to deinstall $SEARX_INSTANCE_NAME?"; then + return + fi + remove_searx_uwsgi + drop_service_account "${SERVICE_USER}" + remove_settings + wait_key + if service_is_available "${PUBLIC_URL}"; then + MSG="** Don't forgett to remove your public site! (${PUBLIC_URL}) **" wait_key 10 + fi +} + +assert_user() { + rst_title "user $SERVICE_USER" section + echo + tee_stderr 1 <<EOF | bash | prefix_stdout +useradd --shell /bin/bash --system \ + --home-dir "$SERVICE_HOME" \ + --comment 'Privacy-respecting metasearch engine' $SERVICE_USER +mkdir "$SERVICE_HOME" +chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME" +groups $SERVICE_USER +EOF + #SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)" + #export SERVICE_HOME + #echo "export SERVICE_HOME=$SERVICE_HOME" +} + +clone_is_available() { + [[ -f "$SEARX_SRC/.git/config" ]] +} + +# shellcheck disable=SC2164 +clone_searx() { + rst_title "Clone searx sources" section + echo + if ! sudo -i -u "$SERVICE_USER" ls -d "$REPO_ROOT" > /dev/null; then + die 42 "user '$SERVICE_USER' missed read permission: $REPO_ROOT" + fi + SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME 2>/dev/null)" + if [[ ! "${SERVICE_HOME}" ]]; then + err_msg "to clone searx sources, user $SERVICE_USER hast to be created first" + return 42 + fi + export SERVICE_HOME + git_clone "$REPO_ROOT" "$SEARX_SRC" \ + "$GIT_BRANCH" "$SERVICE_USER" + + pushd "${SEARX_SRC}" > /dev/null + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +cd "${SEARX_SRC}" +git remote set-url origin ${GIT_URL} +git config user.email "$ADMIN_EMAIL" +git config user.name "$ADMIN_NAME" +git config --list +EOF + popd > /dev/null +} + +install_settings() { + rst_title "${SEARX_SETTINGS_PATH}" section + if ! clone_is_available; then + err_msg "you have to install searx first" + exit 42 + fi + mkdir -p "$(dirname ${SEARX_SETTINGS_PATH})" + + if [[ ! -f ${SEARX_SETTINGS_PATH} ]]; then + info_msg "install settings ${REPO_ROOT}/searx/settings.yml" + info_msg " --> ${SEARX_SETTINGS_PATH}" + cp "${REPO_ROOT}/searx/settings.yml" "${SEARX_SETTINGS_PATH}" + configure_searx + return + fi + + rst_para "Diff between origin's setting file (+) and current (-):" + echo + $DIFF_CMD "${SEARX_SETTINGS_PATH}" "${SEARX_SRC}/searx/settings.yml" + + local action + choose_one action "What should happen to the settings file? " \ + "keep configuration unchanged" \ + "use origin settings" \ + "start interactiv shell" + case $action in + "keep configuration unchanged") + info_msg "leave settings file unchanged" + ;; + "use origin settings") + backup_file "${SEARX_SETTINGS_PATH}" + info_msg "install origin settings" + cp "${SEARX_SRC}/searx/settings.yml" "${SEARX_SETTINGS_PATH}" + ;; + "start interactiv shell") + backup_file "${SEARX_SETTINGS_PATH}" + echo -e "// exit with [${_BCyan}CTRL-D${_creset}]" + sudo -H -i + rst_para 'Diff between new setting file (-) and current (+):' + echo + $DIFF_CMD "${SEARX_SRC}/searx/settings.yml" "${SEARX_SETTINGS_PATH}" + wait_key + ;; + esac +} + +remove_settings() { + rst_title "remove searx settings" section + echo + info_msg "delete ${SEARX_SETTINGS_PATH}" + rm -f "${SEARX_SETTINGS_PATH}" +} + +remove_searx() { + rst_title "Drop searx sources" section + if ask_yn "Do you really want to drop searx sources ($SEARX_SRC)?"; then + rm -rf "$SEARX_SRC" + else + rst_para "Leave searx sources unchanged." + fi +} + +pyenv_is_available() { + [[ -f "${SEARX_PYENV}/bin/activate" ]] +} + +create_pyenv() { + rst_title "Create virtualenv (python)" section + echo + if [[ ! -f "${SEARX_SRC}/manage.sh" ]]; then + err_msg "to create pyenv for searx, searx has to be cloned first" + return 42 + fi + info_msg "create pyenv in ${SEARX_PYENV}" + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +rm -rf "${SEARX_PYENV}" +python3 -m venv "${SEARX_PYENV}" +grep -qFs -- 'source ${SEARX_PYENV}/bin/activate' ~/.profile \ + || echo 'source ${SEARX_PYENV}/bin/activate' >> ~/.profile +EOF + info_msg "inspect python's virtual environment" + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +command -v python && python --version +EOF + wait_key + info_msg "install needed python packages" + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +pip install -U pip +pip install -U setuptools +pip install -U wheel +pip install -U -e . +cd ${SEARX_SRC} +pip install -e . +EOF +} + +remove_pyenv() { + rst_title "Remove virtualenv (python)" section + if ! ask_yn "Do you really want to drop ${SEARX_PYENV} ?"; then + return + fi + info_msg "remove pyenv activation from ~/.profile" + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +grep -v 'source ${SEARX_PYENV}/bin/activate' ~/.profile > ~/.profile.## +mv ~/.profile.## ~/.profile +EOF + rm -rf "${SEARX_PYENV}" +} + +configure_searx() { + rst_title "Configure searx" section + rst_para "Setup searx config located at $SEARX_SETTINGS_PATH" + echo + tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +sed -i -e "s/ultrasecretkey/$(openssl rand -hex 16)/g" "$SEARX_SETTINGS_PATH" +sed -i -e "s/{instance_name}/${SEARX_INSTANCE_NAME}/g" "$SEARX_SETTINGS_PATH" +EOF +} + +test_local_searx() { + rst_title "Testing searx instance localy" section + echo + + if service_is_available "http://${SEARX_INTERNAL_HTTP}" &>/dev/null; then + err_msg "URL/port http://${SEARX_INTERNAL_HTTP} is already in use, you" + err_msg "should stop that service before starting local tests!" + if ! ask_yn "Continue with local tests?"; then + return + fi + fi + sed -i -e "s/debug : False/debug : True/g" "$SEARX_SETTINGS_PATH" + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +export SEARX_SETTINGS_PATH="${SEARX_SETTINGS_PATH}" +cd ${SEARX_SRC} +timeout 10 python searx/webapp.py & +sleep 3 +curl --location --verbose --head --insecure $SEARX_INTERNAL_HTTP +EOF + sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS_PATH" +} + +install_searx_uwsgi() { + rst_title "Install searx's uWSGI app (searx.ini)" section + echo + install_uwsgi + uWSGI_install_app "$SEARX_UWSGI_APP" +} + +remove_searx_uwsgi() { + rst_title "Remove searx's uWSGI app (searx.ini)" section + echo + uWSGI_remove_app "$SEARX_UWSGI_APP" +} + +activate_service() { + rst_title "Activate $SEARX_INSTANCE_NAME (service)" section + echo + uWSGI_enable_app "$SEARX_UWSGI_APP" + uWSGI_restart "$SEARX_UWSGI_APP" +} + +deactivate_service() { + rst_title "De-Activate $SEARX_INSTANCE_NAME (service)" section + echo + uWSGI_disable_app "$SEARX_UWSGI_APP" + uWSGI_restart "$SEARX_UWSGI_APP" +} + +enable_image_proxy() { + info_msg "try to enable image_proxy ..." + tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +sed -i -e "s/image_proxy : False/image_proxy : True/g" "$SEARX_SETTINGS_PATH" +EOF + uWSGI_restart "$SEARX_UWSGI_APP" +} + +disable_image_proxy() { + info_msg "try to enable image_proxy ..." + tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +sed -i -e "s/image_proxy : True/image_proxy : False/g" "$SEARX_SETTINGS_PATH" +EOF + uWSGI_restart "$SEARX_UWSGI_APP" +} + +enable_debug() { + warn_msg "Do not enable debug in production enviroments!!" + info_msg "try to enable debug mode ..." + tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +sed -i -e "s/debug : False/debug : True/g" "$SEARX_SETTINGS_PATH" +EOF + uWSGI_restart "$SEARX_UWSGI_APP" +} + +disable_debug() { + info_msg "try to disable debug mode ..." + tee_stderr 0.1 <<EOF | sudo -H -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS_PATH" +EOF + uWSGI_restart "$SEARX_UWSGI_APP" +} + +set_result_proxy() { + + # usage: set_result_proxy <URL> [<key>] + + info_msg "try to set result proxy: '$1' ($2)" + cp "${SEARX_SETTINGS_PATH}" "${SEARX_SETTINGS_PATH}.bak" + _set_result_proxy "$1" "$2" > "${SEARX_SETTINGS_PATH}" +} + +_set_result_proxy() { + local line + local stage=0 + local url=" url: $1" + local key=" key: !!binary \"$2\"" + if [[ -z $2 ]]; then + key= + fi + + while IFS= read -r line + do + if [[ $stage = 0 ]] || [[ $stage = 2 ]] ; then + if [[ $line =~ ^[[:space:]]*#*[[:space:]]*result_proxy[[:space:]]*:[[:space:]]*$ ]]; then + if [[ $stage = 0 ]]; then + stage=1 + echo "result_proxy:" + continue + elif [[ $stage = 2 ]]; then + continue + fi + fi + fi + if [[ $stage = 1 ]] || [[ $stage = 2 ]] ; then + if [[ $line =~ ^[[:space:]]*#*[[:space:]]*url[[:space:]]*:[[:space:]] ]]; then + [[ $stage = 1 ]] && echo "$url" + continue + elif [[ $line =~ ^[[:space:]]*#*[[:space:]]*key[[:space:]]*:[[:space:]] ]]; then + [[ $stage = 1 ]] && [[ -n $key ]] && echo "$key" + continue + elif [[ $line =~ ^[[:space:]]*$ ]]; then + stage=2 + fi + fi + echo "$line" + done < "${SEARX_SETTINGS_PATH}.bak" +} + +function has_substring() { + [[ "$1" != "${2/$1/}" ]] +} +inspect_service() { + rst_title "service status & log" + cat <<EOF + +sourced ${DOT_CONFIG#"$REPO_ROOT/"} : + + PUBLIC_URL : ${PUBLIC_URL} + SEARX_URL_PATH : ${SEARX_URL_PATH} + SEARX_INSTANCE_NAME : ${SEARX_INSTANCE_NAME} + SEARX_INTERNAL_HTTP : ${SEARX_INTERNAL_HTTP} + +EOF + + if service_account_is_available "$SERVICE_USER"; then + info_msg "Service account $SERVICE_USER exists." + else + err_msg "Service account $SERVICE_USER does not exists!" + fi + + if pyenv_is_available; then + info_msg "~$SERVICE_USER: python environment is available." + else + err_msg "~$SERVICE_USER: python environment is not available!" + fi + + if clone_is_available; then + info_msg "~$SERVICE_USER: Searx software is installed." + else + err_msg "~$SERVICE_USER: Missing searx software!" + fi + + if uWSGI_app_enabled "$SEARX_UWSGI_APP"; then + info_msg "uWSGI app $SEARX_UWSGI_APP is enabled." + else + err_msg "uWSGI app $SEARX_UWSGI_APP not enabled!" + fi + + uWSGI_app_available "$SEARX_UWSGI_APP" \ + || err_msg "uWSGI app $SEARX_UWSGI_APP not available!" + + if in_container; then + lxc_suite_info + else + info_msg "public URL --> ${PUBLIC_URL}" + info_msg "internal URL --> http://${SEARX_INTERNAL_HTTP}" + fi + + if ! service_is_available "http://${SEARX_INTERNAL_HTTP}"; then + err_msg "uWSGI app (service) at http://${SEARX_INTERNAL_HTTP} is not available!" + MSG="${_Green}[${_BCyan}CTRL-C${_Green}] to stop or [${_BCyan}KEY${_Green}] to continue"\ + wait_key + fi + + if ! service_is_available "${PUBLIC_URL}"; then + warn_msg "Public service at ${PUBLIC_URL} is not available!" + if ! in_container; then + warn_msg "Check if public name is correct and routed or use the public IP from above." + fi + fi + + local _debug_on + if ask_yn "Enable searx debug mode?"; then + enable_debug + _debug_on=1 + fi + echo + + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) + systemctl --no-pager -l status "${SERVICE_NAME}" + ;; + arch-*) + systemctl --no-pager -l status "uwsgi@${SERVICE_NAME%.*}" + ;; + fedora-*) + systemctl --no-pager -l status uwsgi + ;; + esac + + # shellcheck disable=SC2059 + printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log" + read -r -s -n1 -t 5 + echo + + while true; do + trap break 2 + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) tail -f /var/log/uwsgi/app/searx.log ;; + arch-*) journalctl -f -u "uwsgi@${SERVICE_NAME%.*}" ;; + fedora-*) journalctl -f -u uwsgi ;; + esac + done + + if [[ $_debug_on == 1 ]]; then + disable_debug + fi + return 0 +} + +install_apache_site() { + rst_title "Install Apache site $APACHE_SEARX_SITE" + + rst_para "\ +This installs the searx uwsgi app as apache site. If your server is public to +the internet, you should instead use a reverse proxy (filtron) to block +excessively bot queries." + + ! apache_is_installed && err_msg "Apache is not installed." + + if ! ask_yn "Do you really want to continue?" Yn; then + return + else + install_apache + fi + + apache_install_site --variant=uwsgi "${APACHE_SEARX_SITE}" + + rst_title "Install searx's uWSGI app (searx.ini)" section + echo + uWSGI_install_app --variant=socket "$SEARX_UWSGI_APP" + + if ! service_is_available "${PUBLIC_URL}"; then + err_msg "Public service at ${PUBLIC_URL} is not available!" + fi +} + +remove_apache_site() { + + rst_title "Remove Apache site ${APACHE_SEARX_SITE}" + + rst_para "\ +This removes apache site ${APACHE_SEARX_SITE}." + + ! apache_is_installed && err_msg "Apache is not installed." + + if ! ask_yn "Do you really want to continue?" Yn; then + return + fi + + apache_remove_site "${APACHE_SEARX_SITE}" + + rst_title "Remove searx's uWSGI app (searx.ini)" section + echo + uWSGI_remove_app "$SEARX_UWSGI_APP" +} + +rst-doc() { + local debian="${SEARX_PACKAGES_debian}" + local arch="${SEARX_PACKAGES_arch}" + local fedora="${SEARX_PACKAGES_fedora}" + local debian_build="${BUILD_PACKAGES_debian}" + local arch_build="${BUILD_PACKAGES_arch}" + local fedora_build="${BUILD_PACKAGES_fedora}" + debian="$(echo "${debian}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + arch="$(echo "${arch}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + fedora="$(echo "${fedora}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + debian_build="$(echo "${debian_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + arch_build="$(echo "${arch_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + fedora_build="$(echo "${fedora_build}" | sed 's/.*/ & \\/' | sed '$ s/.$//')" + + eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/searx.rst")\"" + + # I use ubuntu-20.04 here to demonstrate that versions are also suported, + # normaly debian-* and ubuntu-* are most the same. + + for DIST_NAME in ubuntu-20.04 arch fedora; do + ( + DIST_ID=${DIST_NAME%-*} + DIST_VERS=${DIST_NAME#*-} + [[ $DIST_VERS =~ $DIST_ID ]] && DIST_VERS= + uWSGI_distro_setup + + echo -e "\n.. START searx uwsgi-description $DIST_NAME" + + case $DIST_ID-$DIST_VERS in + ubuntu-*|debian-*) cat <<EOF +# init.d --> /usr/share/doc/uwsgi/README.Debian.gz +# For uWSGI debian uses the LSB init process, this might be changed +# one day, see https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=833067 + +create ${uWSGI_APPS_AVAILABLE}/${SEARX_UWSGI_APP} +enable: sudo -H ln -s ${uWSGI_APPS_AVAILABLE}/${SEARX_UWSGI_APP} ${uWSGI_APPS_ENABLED}/ +start: sudo -H service uwsgi start ${SEARX_UWSGI_APP%.*} +restart: sudo -H service uwsgi restart ${SEARX_UWSGI_APP%.*} +stop: sudo -H service uwsgi stop ${SEARX_UWSGI_APP%.*} +disable: sudo -H rm ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP} +EOF + ;; + arch-*) cat <<EOF +# systemd --> /usr/lib/systemd/system/uwsgi@.service +# For uWSGI archlinux uses systemd template units, see +# - http://0pointer.de/blog/projects/instances.html +# - https://uwsgi-docs.readthedocs.io/en/latest/Systemd.html#one-service-per-app-in-systemd + +create: ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP} +enable: sudo -H systemctl enable uwsgi@${SEARX_UWSGI_APP%.*} +start: sudo -H systemctl start uwsgi@${SEARX_UWSGI_APP%.*} +restart: sudo -H systemctl restart uwsgi@${SEARX_UWSGI_APP%.*} +stop: sudo -H systemctl stop uwsgi@${SEARX_UWSGI_APP%.*} +disable: sudo -H systemctl disable uwsgi@${SEARX_UWSGI_APP%.*} +EOF + ;; + fedora-*) cat <<EOF +# systemd --> /usr/lib/systemd/system/uwsgi.service +# The unit file starts uWSGI in emperor mode (/etc/uwsgi.ini), see +# - https://uwsgi-docs.readthedocs.io/en/latest/Emperor.html + +create: ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP} +restart: sudo -H touch ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP} +disable: sudo -H rm ${uWSGI_APPS_ENABLED}/${SEARX_UWSGI_APP} +EOF + ;; + esac + echo -e ".. END searx uwsgi-description $DIST_NAME" + + echo -e "\n.. START searx uwsgi-appini $DIST_NAME" + eval "echo \"$(< "${TEMPLATES}/${uWSGI_APPS_AVAILABLE}/${SEARX_UWSGI_APP}")\"" + echo -e "\n.. END searx uwsgi-appini $DIST_NAME" + + ) + done + +} + +# ---------------------------------------------------------------------------- +main "$@" +# ---------------------------------------------------------------------------- diff --git a/utils/site-python/sphinx_build_tools.py b/utils/site-python/sphinx_build_tools.py new file mode 100644 index 000000000..b9ebdeacc --- /dev/null +++ b/utils/site-python/sphinx_build_tools.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8; mode: python -*- +"""Implement some sphinx-build tools. + +""" + +import os +import sys +from sphinx.util.pycompat import execfile_ + +# ------------------------------------------------------------------------------ +def load_sphinx_config(namespace): +# ------------------------------------------------------------------------------ + + u"""Load an additional configuration file into *namespace*. + + The name of the configuration file is taken from the environment + ``SPHINX_CONF``. The external configuration file extends (or overwrites) the + configuration values from the origin ``conf.py``. With this you are able to + maintain *build themes*. To your docs/conf.py add:: + + from sphinx_build_tools import load_sphinx_config + ... + + # Since loadConfig overwrites settings from the global namespace, it has to be + # the last statement in the conf.py file + + load_sphinx_config(globals()) + + """ + + config_file = os.environ.get("SPHINX_CONF", None) + if (config_file is not None + and os.path.normpath(namespace["__file__"]) != os.path.normpath(config_file) ): + config_file = os.path.abspath(config_file) + + if os.path.isfile(config_file): + sys.stdout.write( + "load additional sphinx-config: %s\n" + % config_file) + config = namespace.copy() + config['__file__'] = config_file + execfile_(config_file, config) + del config['__file__'] + namespace.update(config) + else: + sys.stderr.write( + "WARNING: additional sphinx-config not found: %s\n" + % config_file) diff --git a/utils/templates/etc/apache2 b/utils/templates/etc/apache2 new file mode 120000 index 000000000..558a90717 --- /dev/null +++ b/utils/templates/etc/apache2 @@ -0,0 +1 @@ +httpd
\ No newline at end of file diff --git a/utils/templates/etc/filtron/rules.json b/utils/templates/etc/filtron/rules.json new file mode 100644 index 000000000..fff70fa8f --- /dev/null +++ b/utils/templates/etc/filtron/rules.json @@ -0,0 +1,129 @@ +[ + { + "name": "roboagent limit", + "filters": [ + "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client|Ruby|UniversalFeedParser)" + ], + "limit": 0, + "stop": true, + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "botlimit", + "filters": [ + "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)" + ], + "limit": 0, + "stop": true, + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "suspiciously frequent IP", + "filters": [], + "interval": 600, + "limit": 30, + "aggregations": [ + "Header:X-Forwarded-For" + ], + "actions":[ + {"name":"log"} + ] + }, + { + "name": "search request", + "filters": [ + "Param:q", + "Path=^(/|/search)$" + ], + "interval": 61, + "limit": 999, + "subrules": [ + { + "name": "missing Accept-Language", + "filters": ["!Header:Accept-Language"], + "limit": 0, + "stop": true, + "actions": [ + {"name":"log"}, + {"name": "block", + "params": {"message": "Rate limit exceeded"}} + ] + }, + { + "name": "suspiciously Connection=close header", + "filters": ["Header:Connection=close"], + "limit": 0, + "stop": true, + "actions": [ + {"name":"log"}, + {"name": "block", + "params": {"message": "Rate limit exceeded"}} + ] + }, + { + "name": "IP limit", + "interval": 61, + "limit": 9, + "stop": true, + "aggregations": [ + "Header:X-Forwarded-For" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "rss/json limit", + "filters": [ + "Param:format=(csv|json|rss)" + ], + "interval": 121, + "limit": 2, + "stop": true, + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "useragent limit", + "interval": 61, + "limit": 199, + "aggregations": [ + "Header:User-Agent" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + } + ] + } +] diff --git a/utils/templates/etc/httpd/sites-available/morty.conf b/utils/templates/etc/httpd/sites-available/morty.conf new file mode 100644 index 000000000..daeb3635a --- /dev/null +++ b/utils/templates/etc/httpd/sites-available/morty.conf @@ -0,0 +1,28 @@ +# -*- coding: utf-8; mode: apache -*- + +LoadModule headers_module ${APACHE_MODULES}/mod_headers.so +LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so +LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so +#LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so + +# SetEnvIf Request_URI "${PUBLIC_URL_PATH_MORTY}" dontlog +# CustomLog /dev/null combined env=dontlog + +<Location ${PUBLIC_URL_PATH_MORTY} > + + <IfModule mod_security2.c> + SecRuleEngine Off + </IfModule> + + Require all granted + + Order deny,allow + Deny from all + #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + ProxyPreserveHost On + ProxyPass http://${MORTY_LISTEN} + RequestHeader set X-Script-Name ${PUBLIC_URL_PATH_MORTY} + +</Location> diff --git a/utils/templates/etc/httpd/sites-available/searx.conf:filtron b/utils/templates/etc/httpd/sites-available/searx.conf:filtron new file mode 100644 index 000000000..379d47e24 --- /dev/null +++ b/utils/templates/etc/httpd/sites-available/searx.conf:filtron @@ -0,0 +1,33 @@ +# -*- coding: utf-8; mode: apache -*- + +LoadModule headers_module ${APACHE_MODULES}/mod_headers.so +LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so +LoadModule proxy_http_module ${APACHE_MODULES}/mod_proxy_http.so +#LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so + +# SetEnvIf Request_URI "${FILTRON_URL_PATH}" dontlog +# CustomLog /dev/null combined env=dontlog + +# SecRuleRemoveById 981054 +# SecRuleRemoveById 981059 +# SecRuleRemoveById 981060 +# SecRuleRemoveById 950907 + +<Location ${FILTRON_URL_PATH} > + + <IfModule mod_security2.c> + SecRuleEngine Off + </IfModule> + + Require all granted + + Order deny,allow + Deny from all + #Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + ProxyPreserveHost On + ProxyPass http://${FILTRON_LISTEN} + RequestHeader set X-Script-Name ${FILTRON_URL_PATH} + +</Location> diff --git a/utils/templates/etc/httpd/sites-available/searx.conf:uwsgi b/utils/templates/etc/httpd/sites-available/searx.conf:uwsgi new file mode 100644 index 000000000..ef702de3a --- /dev/null +++ b/utils/templates/etc/httpd/sites-available/searx.conf:uwsgi @@ -0,0 +1,27 @@ +# -*- coding: utf-8; mode: apache -*- + +LoadModule headers_module ${APACHE_MODULES}/mod_headers.so +LoadModule proxy_module ${APACHE_MODULES}/mod_proxy.so +LoadModule proxy_uwsgi_module ${APACHE_MODULES}/mod_proxy_uwsgi.so +# LoadModule setenvif_module ${APACHE_MODULES}/mod_setenvif.so + +# SetEnvIf Request_URI "${SEARX_URL_PATH}" dontlog +# CustomLog /dev/null combined env=dontlog + +<Location ${SEARX_URL_PATH}> + + <IfModule mod_security2.c> + SecRuleEngine Off + </IfModule> + + Require all granted + + Order deny,allow + Deny from all + # Allow from fd00::/8 192.168.0.0/16 fe80::/10 127.0.0.0/8 ::1 + Allow from all + + ProxyPreserveHost On + ProxyPass unix:${SEARX_UWSGI_SOCKET}|uwsgi://uwsgi-uds-searx/ + +</Location> diff --git a/utils/templates/etc/nginx/default.apps-available/morty.conf b/utils/templates/etc/nginx/default.apps-available/morty.conf new file mode 100644 index 000000000..e7ffa27e2 --- /dev/null +++ b/utils/templates/etc/nginx/default.apps-available/morty.conf @@ -0,0 +1,11 @@ +# https://example.org/morty + +location /morty { + proxy_pass http://127.0.0.1:3000/; + + proxy_set_header Host \$http_host; + proxy_set_header Connection \$http_connection; + proxy_set_header X-Real-IP \$remote_addr; + proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; + proxy_set_header X-Scheme \$scheme; +} diff --git a/utils/templates/etc/nginx/default.apps-available/searx.conf:filtron b/utils/templates/etc/nginx/default.apps-available/searx.conf:filtron new file mode 100644 index 000000000..d3137e42d --- /dev/null +++ b/utils/templates/etc/nginx/default.apps-available/searx.conf:filtron @@ -0,0 +1,16 @@ +# https://example.org/searx + +location ${SEARX_URL_PATH} { + proxy_pass http://127.0.0.1:4004/; + + proxy_set_header Host \$http_host; + proxy_set_header Connection \$http_connection; + proxy_set_header X-Real-IP \$remote_addr; + proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; + proxy_set_header X-Scheme \$scheme; + proxy_set_header X-Script-Name ${SEARX_URL_PATH}; +} + +location ${SEARX_URL_PATH}/static { + alias ${SEARX_SRC}/searx/static; +} diff --git a/utils/templates/etc/uwsgi/apps-archlinux/searx.ini b/utils/templates/etc/uwsgi/apps-archlinux/searx.ini new file mode 100644 index 000000000..51f659d0f --- /dev/null +++ b/utils/templates/etc/uwsgi/apps-archlinux/searx.ini @@ -0,0 +1,80 @@ +[uwsgi] + +# uWSGI core +# ---------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core + +# Who will run the code +uid = ${SERVICE_USER} +gid = ${SERVICE_GROUP} + +# chdir to specified directory before apps loading +chdir = ${SEARX_SRC}/searx + +# searx configuration (settings.yml) +env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH} + +# disable logging for privacy +logger = systemd +disable-logging = true + +# The right granted on the created socket +chmod-socket = 666 + +# Plugin to use and interpretor config +single-interpreter = true + +# enable master process +master = true + +# load apps in each worker instead of the master +lazy-apps = true + +# load uWSGI plugins +plugin = python + +# By default the Python plugin does not initialize the GIL. This means your +# app-generated threads will not run. If you need threads, remember to enable +# them with enable-threads. Running uWSGI in multithreading mode (with the +# threads options) will automatically enable threading support. This *strange* +# default behaviour is for performance reasons. +enable-threads = true + + +# plugin: python +# -------------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python + +# load a WSGI module +module = searx.webapp + +# set PYTHONHOME/virtualenv +virtualenv = ${SEARX_PYENV} + +# add directory (or glob) to pythonpath +pythonpath = ${SEARX_SRC} + + +# speak to upstream +# ----------------- +# +# Activate the 'http' configuration for filtron or activate the 'socket' +# configuration if you setup your HTTP server to use uWSGI protocol via sockets. + +# using IP: +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http +# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html + +http = ${SEARX_INTERNAL_HTTP} + +# using unix-sockets: +# +# On some distributions you need to create the app folder for the sockets:: +# +# mkdir -p /run/uwsgi/app/searx +# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx +# +# socket = /run/uwsgi/app/searx/socket
\ No newline at end of file diff --git a/utils/templates/etc/uwsgi/apps-archlinux/searx.ini:socket b/utils/templates/etc/uwsgi/apps-archlinux/searx.ini:socket new file mode 100644 index 000000000..eeabb3715 --- /dev/null +++ b/utils/templates/etc/uwsgi/apps-archlinux/searx.ini:socket @@ -0,0 +1,80 @@ +[uwsgi] + +# uWSGI core +# ---------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core + +# Who will run the code +uid = ${SERVICE_USER} +gid = ${SERVICE_GROUP} + +# chdir to specified directory before apps loading +chdir = ${SEARX_SRC}/searx + +# searx configuration (settings.yml) +env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH} + +# disable logging for privacy +logger = systemd +disable-logging = true + +# The right granted on the created socket +chmod-socket = 666 + +# Plugin to use and interpretor config +single-interpreter = true + +# enable master process +master = true + +# load apps in each worker instead of the master +lazy-apps = true + +# load uWSGI plugins +plugin = python + +# By default the Python plugin does not initialize the GIL. This means your +# app-generated threads will not run. If you need threads, remember to enable +# them with enable-threads. Running uWSGI in multithreading mode (with the +# threads options) will automatically enable threading support. This *strange* +# default behaviour is for performance reasons. +enable-threads = true + + +# plugin: python +# -------------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python + +# load a WSGI module +module = searx.webapp + +# set PYTHONHOME/virtualenv +virtualenv = ${SEARX_PYENV} + +# add directory (or glob) to pythonpath +pythonpath = ${SEARX_SRC} + + +# speak to upstream +# ----------------- +# +# Activate the 'http' configuration for filtron or activate the 'socket' +# configuration if you setup your HTTP server to use uWSGI protocol via sockets. + +# using IP: +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http +# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html + +# http = ${SEARX_INTERNAL_HTTP} + +# using unix-sockets: +# +# On some distributions you need to create the app folder for the sockets:: +# +# mkdir -p /run/uwsgi/app/searx +# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx +# +socket = /run/uwsgi/app/searx/socket
\ No newline at end of file diff --git a/utils/templates/etc/uwsgi/apps-available/searx.ini b/utils/templates/etc/uwsgi/apps-available/searx.ini new file mode 100644 index 000000000..9785d7cd1 --- /dev/null +++ b/utils/templates/etc/uwsgi/apps-available/searx.ini @@ -0,0 +1,79 @@ +[uwsgi] + +# uWSGI core +# ---------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core + +# Who will run the code +uid = ${SERVICE_USER} +gid = ${SERVICE_GROUP} + +# chdir to specified directory before apps loading +chdir = ${SEARX_SRC}/searx + +# searx configuration (settings.yml) +env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH} + +# disable logging for privacy +disable-logging = true + +# The right granted on the created socket +chmod-socket = 666 + +# Plugin to use and interpretor config +single-interpreter = true + +# enable master process +master = true + +# load apps in each worker instead of the master +lazy-apps = true + +# load uWSGI plugins +plugin = python3,http + +# By default the Python plugin does not initialize the GIL. This means your +# app-generated threads will not run. If you need threads, remember to enable +# them with enable-threads. Running uWSGI in multithreading mode (with the +# threads options) will automatically enable threading support. This *strange* +# default behaviour is for performance reasons. +enable-threads = true + + +# plugin: python +# -------------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python + +# load a WSGI module +module = searx.webapp + +# set PYTHONHOME/virtualenv +virtualenv = ${SEARX_PYENV} + +# add directory (or glob) to pythonpath +pythonpath = ${SEARX_SRC} + + +# speak to upstream +# ----------------- +# +# Activate the 'http' configuration for filtron or activate the 'socket' +# configuration if you setup your HTTP server to use uWSGI protocol via sockets. + +# using IP: +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http +# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html + +http = ${SEARX_INTERNAL_HTTP} + +# using unix-sockets: +# +# On some distributions you need to create the app folder for the sockets:: +# +# mkdir -p /run/uwsgi/app/searx +# chmod -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx +# +# socket = /run/uwsgi/app/searx/socket
\ No newline at end of file diff --git a/utils/templates/etc/uwsgi/apps-available/searx.ini:socket b/utils/templates/etc/uwsgi/apps-available/searx.ini:socket new file mode 100644 index 000000000..88436e5eb --- /dev/null +++ b/utils/templates/etc/uwsgi/apps-available/searx.ini:socket @@ -0,0 +1,79 @@ +[uwsgi] + +# uWSGI core +# ---------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#uwsgi-core + +# Who will run the code +uid = ${SERVICE_USER} +gid = ${SERVICE_GROUP} + +# chdir to specified directory before apps loading +chdir = ${SEARX_SRC}/searx + +# searx configuration (settings.yml) +env = SEARX_SETTINGS_PATH=${SEARX_SETTINGS_PATH} + +# disable logging for privacy +disable-logging = true + +# The right granted on the created socket +chmod-socket = 666 + +# Plugin to use and interpretor config +single-interpreter = true + +# enable master process +master = true + +# load apps in each worker instead of the master +lazy-apps = true + +# load uWSGI plugins +plugin = python3,http + +# By default the Python plugin does not initialize the GIL. This means your +# app-generated threads will not run. If you need threads, remember to enable +# them with enable-threads. Running uWSGI in multithreading mode (with the +# threads options) will automatically enable threading support. This *strange* +# default behaviour is for performance reasons. +enable-threads = true + + +# plugin: python +# -------------- +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-python + +# load a WSGI module +module = searx.webapp + +# set PYTHONHOME/virtualenv +virtualenv = ${SEARX_PYENV} + +# add directory (or glob) to pythonpath +pythonpath = ${SEARX_SRC} + + +# speak to upstream +# ----------------- +# +# Activate the 'http' configuration for filtron or activate the 'socket' +# configuration if you setup your HTTP server to use uWSGI protocol via sockets. + +# using IP: +# +# https://uwsgi-docs.readthedocs.io/en/latest/Options.html#plugin-http +# Native HTTP support: https://uwsgi-docs.readthedocs.io/en/latest/HTTP.html + +# http = ${SEARX_INTERNAL_HTTP} + +# using unix-sockets: +# +# On some distributions you need to create the app folder for the sockets:: +# +# mkdir -p /run/uwsgi/app/searx +# chown -R ${SERVICE_USER}:${SERVICE_GROUP} /run/uwsgi/app/searx +# +socket = /run/uwsgi/app/searx/socket
\ No newline at end of file diff --git a/utils/templates/lib/systemd/system/filtron.service b/utils/templates/lib/systemd/system/filtron.service new file mode 100644 index 000000000..3b0c6edcc --- /dev/null +++ b/utils/templates/lib/systemd/system/filtron.service @@ -0,0 +1,29 @@ +[Unit] + +Description=${SERVICE_NAME} +After=syslog.target +After=network.target + +[Service] + +Type=simple +User=${SERVICE_USER} +Group=${SERVICE_GROUP} +WorkingDirectory=${SERVICE_HOME} +ExecStart=${SERVICE_HOME}/go-apps/bin/filtron -api '${FILTRON_API}' -listen '${FILTRON_LISTEN}' -rules '${FILTRON_RULES}' -target '${FILTRON_TARGET}' + +Restart=always +Environment=USER=${SERVICE_USER} HOME=${SERVICE_HOME} + +# Some distributions may not support these hardening directives. If you cannot +# start the service due to an unknown option, comment out the ones not supported +# by your version of systemd. + +ProtectSystem=full +PrivateDevices=yes +PrivateTmp=yes +NoNewPrivileges=true + +[Install] + +WantedBy=multi-user.target diff --git a/utils/templates/lib/systemd/system/morty.service b/utils/templates/lib/systemd/system/morty.service new file mode 100644 index 000000000..25b676b51 --- /dev/null +++ b/utils/templates/lib/systemd/system/morty.service @@ -0,0 +1,29 @@ +[Unit] + +Description=${SERVICE_NAME} +After=syslog.target +After=network.target + +[Service] + +Type=simple +User=${SERVICE_USER} +Group=${SERVICE_GROUP} +WorkingDirectory=${SERVICE_HOME} +ExecStart=${SERVICE_HOME}/go-apps/bin/morty -key '${MORTY_KEY}' -listen '${MORTY_LISTEN}' -timeout ${MORTY_TIMEOUT} + +Restart=always +Environment=USER=${SERVICE_USER} HOME=${SERVICE_HOME} DEBUG=${SERVICE_ENV_DEBUG} + +# Some distributions may not support these hardening directives. If you cannot +# start the service due to an unknown option, comment out the ones not supported +# by your version of systemd. + +ProtectSystem=full +PrivateDevices=yes +PrivateTmp=yes +NoNewPrivileges=true + +[Install] + +WantedBy=multi-user.target |