diff options
| -rw-r--r-- | Makefile | 1 | ||||
| -rwxr-xr-x | utils/filtron.sh | 3 | ||||
| -rwxr-xr-x | utils/lib.sh | 186 | ||||
| -rwxr-xr-x | utils/searx.sh | 349 |
4 files changed, 532 insertions, 7 deletions
@@ -79,6 +79,7 @@ test: test.pep8 test.unit test.sh test.robot test.sh: shellcheck -x utils/lib.sh shellcheck -x utils/filtron.sh + shellcheck -x utils/searx.sh test.pep8: pyenvinstall $(PY_ENV_ACT); ./manage.sh pep8_check diff --git a/utils/filtron.sh b/utils/filtron.sh index 5c8a738b0..50cce6e65 100755 --- a/utils/filtron.sh +++ b/utils/filtron.sh @@ -192,7 +192,8 @@ assert_user() { rst_title "user $SERVICE_USER" section echo tee_stderr 1 <<EOF | bash | prefix_stdout -sudo -H adduser --shell /bin/bash --system --home $SERVICE_HOME --group --gecos 'Filtron' $SERVICE_USER +sudo -H adduser --shell /bin/bash --system --home $SERVICE_HOME \ + --disabled-password --group --gecos 'Filtron' $SERVICE_USER sudo -H usermod -a -G shadow $SERVICE_USER groups $SERVICE_USER EOF diff --git a/utils/lib.sh b/utils/lib.sh index fd6b92129..f43a117cb 100755 --- a/utils/lib.sh +++ b/utils/lib.sh @@ -2,6 +2,12 @@ # -*- coding: utf-8; mode: sh -*- # shellcheck disable=SC2059,SC1117,SC2162,SC2004 +ADMIN_NAME="${ADMIN_NAME:-$(git config user.name)}" +ADMIN_NAME="${ADMIN_NAME:-$USER}" + +ADMIN_EMAIL="${ADMIN_EMAIL:-$(git config user.email)}" +ADMIN_EMAIL="${ADMIN_EMAIL:-$USER@$(hostname)}" + if [[ -z "${REPO_ROOT}" ]]; then REPO_ROOT=$(dirname "${BASH_SOURCE[0]}") while [ -h "${REPO_ROOT}" ] ; do @@ -115,7 +121,7 @@ ask_yn() { esac echo while true; do - clean_stdin + clean_stdin printf "$1 ${choice} " # shellcheck disable=SC2086 read -n1 $_t @@ -240,7 +246,7 @@ choose_one() { fi done while true; do - clean_stdin + clean_stdin printf "$1 [$default] " if (( 10 > $max )); then @@ -333,7 +339,7 @@ install_template() { info_msg "install: ${template_file}" sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \ "${template_file}" "${dst}" | prefix_stdout - break + break ;; "leave file unchanged") break @@ -343,12 +349,180 @@ install_template() { echo "// exit with CTRL-D" sudo -H -u "${owner}" -i $DIFF_CMD "${dst}" "${template_file}" - if ask_yn "did you edit ${template_file} to your needs?"; then - break - fi + if ask_yn "did you edit ${template_file} to your needs?"; then + break + fi ;; "diff files") $DIFF_CMD "${dst}" "${template_file}" | prefix_stdout esac done } + + +# uWSGI +# ----- + +uWSGI_SETUP="${uWSGI_SETUP:=/etc/uwsgi}" + +uWSGI_restart() { + + # usage: uWSGI_restart() + + info_msg "restart uWSGI service" + sudo -H systemctl restart uwsgi +} + +uWSGI_install_app() { + + # usage: uWSGI_install_app [--no-eval] /etc/uwsgi/apps-available/myapp.ini ... + + local do_eval="" + local CONF + + if [[ "$1" == "--no-eval" ]]; then + no_eval=$1; shift + fi + + for CONF in "$@"; do + install_template "$no_eval" "${CONF}" root root 644 + uWSGI_enable_app "$(basename "${CONF}")" + info_msg "enabled uWSGI app: $(basename "${CONF}")" + done + uWSGI_restart +} + +uWSGI_remove_app() { + + # usage: uWSGI_remove_app <path.ini> ... + + local CONF + for CONF in "$@"; do + uWSGI_disable_app "$(basename "${CONF}")" + rm -f "$CONF" + info_msg "removed uWSGI app: $(basename "${CONF}")" + done + uWSGI_restart +} + +# shellcheck disable=SC2164 +uWSGI_enable_app() { + + # usage: uWSGI_enable_app <path.ini> + + local CONF=$1 + if [[ -z $CONF ]]; then + err_msg "uWSGI_enable_app missing arguments" + return 42 + fi + pushd "${uWSGI_SETUP}/apps-enabled" >/dev/null + # shellcheck disable=SC2226 + ln -s "../apps-available/$(basename "${CONF}")" + info_msg "enabled uWSGI app: $(basename "${CONF}") (restart uWSGI required)" + popd >/dev/null +} + +uWSGI_disable_app() { + + # usage: uWSGI_disable_app <path.ini> + + local CONF=$1 + if [[ -z $CONF ]]; then + err_msg "uWSGI_enable_app missing arguments" + return 42 + fi + + rm -f "${uWSGI_SETUP}/apps-enabled/$CONF" + info_msg "disabled uWSGI app: $(basename "${CONF}") (restart uWSGI required)" +} + +# distro's package manager +# ------------------------ +# +# FIXME: Arch Linux & RHEL should be added +# + +pkg_install() { + + # usage: TITEL='install foobar' pkg_install foopkg barpkg + + rst_title "${TITLE:-installation of packages}" section + echo -en "\npackage(s)::\n\n $*\n" | $FMT + + if ! ask_yn "Should packages be installed?" Yn 30; then + return 42 + fi + # shellcheck disable=SC2068 + apt-get install -y $@ + wait_key 30 +} + +pkg_remove() { + + # usage: TITEL='remove foobar' pkg_remove foopkg barpkg + + rst_title "${TITLE:-remove packages}" section + echo -en "\npackage(s)::\n\n $*\n" | $FMT + + if ! ask_yn "Should packages be removed (purge)?" Yn 30; then + return 42 + fi + apt-get purge --autoremove --ignore-missing -y "$@" + wait_key 30 +} + +pkg_is_installed() { + + # usage: pkg_is_install foopkg || pkg_install foopkg + + dpkg -l "$1" &> /dev/null + return $? +} + +# git tooling +# ----------- + +# shellcheck disable=SC2164 +git_clone() { + + # usage: + # + # git_clone <url> <name> [<branch> [<user>]] + # git_clone <url> <path> [<branch> [<user>]] + # + # First form uses $CACHE/<name> as destination folder, second form clones + # into <path>. If repository is allready cloned, merge from origin and + # update working tree (if needed, the caller has to stash local changes). + # + # git clone https://github.com/asciimoo/searx searx-src origin/master searxlogin + # + + local url="$1" + local dest="$2" + local branch="$3" + local user="$4" + local prefix="" + + if [[ ! "${dest:0:1}" = "/" ]]; then + dest="$CACHE/$dest" + fi + + [[ -z $branch ]] && branch=master + [[ -z $user ]] && [[ ! -z "${SUDO_USER}" ]] && user="${SUDO_USER}" + [[ -z $user ]] && prefix="sudo -H -u $user" + + if [[ -d "${dest}" ]] ; then + info_msg "already cloned: $dest" + pushd "${dest}" > /dev/null + $prefix git checkout -b "$(basename "$branch")" --track "$branch" + $prefix git pull --all + popd > /dev/null + + else + info_msg "clone into: $dest" + $prefix mkdir -p "$(dirname "$dest")" + pushd "${dest}" > /dev/null + git clone "$url" "$(basename "$dest")" + popd > /dev/null + fi +} diff --git a/utils/searx.sh b/utils/searx.sh new file mode 100755 index 000000000..0d47820b0 --- /dev/null +++ b/utils/searx.sh @@ -0,0 +1,349 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh -*- +# shellcheck disable=SC2119 + +# shellcheck source=utils/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +# ---------------------------------------------------------------------------- +# config +# ---------------------------------------------------------------------------- + +SERVICE_NAME="searx" +SERVICE_USER="${SERVICE_NAME}" +# shellcheck disable=SC2034 +SERVICE_GROUP="${SERVICE_USER}" +SERVICE_HOME="/home/${SERVICE_USER}" + +SEARX_GIT_URL="https://github.com/asciimoo/searx.git" +SEARX_GIT_BRANCH="origin/master" + +# FIXME: Arch Linux & RHEL should be added + +SEARX_APT_PACKAGES="\ +libapache2-mod-uwsgi uwsgi uwsgi-plugin-python3 \ + git build-essential libxslt-dev python3-dev python3-babel zlib1g-dev \ + libffi-dev libssl-dev" + +SEARX_VENV="${SEARX_HOME}/searx-venv" +SEARX_SRC="${SEARX_HOME}/searx-src" +SEARX_SETTINGS="${SEARX_SRC}/searx/settings.yml" +SEARX_INSTANCE_NAME="${SEARX_INSTANCE_NAME:-searx@$(uname -n)}" +SEARX_UWSGI_APP="${uWSGI_SETUP}/apps-available/searx.ini" + +# shellcheck disable=SC2034 +CONFIG_FILES=( + "${SEARX_UWSGI_APP}" +) + +# shellcheck disable=SC2034 +CONFIG_BACKUP_ENCRYPTED=( + "${SEARX_SETTINGS}" +) + +# ---------------------------------------------------------------------------- +usage(){ +# ---------------------------------------------------------------------------- + + # shellcheck disable=SC1117 + cat <<EOF + +usage: + + $(basename "$0") shell + $(basename "$0") install [all|user] + $(basename "$0") update [searx] + $(basename "$0") remove [all] + $(basename "$0") activate [service] + $(basename "$0") deactivate [service] + $(basename "$0") show [service] + +shell + start interactive shell from user ${SERVICE_USER} +install / remove all + complete setup of searx service +update searx + Update searx installation of user ${SERVICE_USER} +activate + activate and start service daemon (systemd unit) +deactivate service + stop and deactivate service daemon (systemd unit) +install user + add service user '$SERVICE_USER' at $SERVICE_HOME +show service + show service status and log +EOF + [ ! -z ${1+x} ] && echo -e "$1" +} + +main(){ + rst_title "$SERVICE_NAME" part + + local _usage="ERROR: unknown or missing $1 command $2" + + case $1 in + --source-only) ;; + -h|--help) usage; exit 0;; + + shell) + sudo_or_exit + interactive_shell + ;; + show) + case $2 in + service) + sudo_or_exit + show_service + ;; + *) usage "$_usage"; exit 42;; + esac ;; + install) + sudo_or_exit + case $2 in + all) install_all ;; + user) assert_user ;; + *) usage "$_usage"; exit 42;; + esac ;; + update) + sudo_or_exit + case $2 in + searx) update_searx;; + *) usage "$_usage"; exit 42;; + esac ;; + remove) + sudo_or_exit + case $2 in + all) remove_all;; + user) remove_user ;; + *) usage "$_usage"; exit 42;; + esac ;; + activate) + sudo_or_exit + case $2 in + service) activate_service ;; + *) usage "$_usage"; exit 42;; + esac ;; + deactivate) + sudo_or_exit + case $2 in + service) deactivate_service ;; + *) usage "$_usage"; exit 42;; + esac ;; + *) usage "ERROR: unknown or missing command $1"; exit 42;; + esac +} + +_service_prefix=" |$SERVICE_USER| " + +install_all() { + rst_title "Install $SERVICE_NAME (service)" + pkg_install "$SEARX_APT_PACKAGES" + wait_key + assert_user + wait_key + clone_searx + wait_key + create_venv + wait_key + configure_searx + wait_key + test_local_searx + wait_key + install_searx_uwsgi + wait_key + + # ToDo ... + # install_apache_site + # test_public_searx + # info_msg "searX --> https://${SEARX_APACHE_DOMAIN}${SEARX_APACHE_URL}" + +} + +update_searx() { + rst_title "Update searx instance" + + echo + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +cp -f ${SEARX_SETTINGS} ${SEARX_SETTINGS}.backup +git stash push -m "BACKUP -- 'update server' at ($(date))" +git checkout -b "$(basename "$SEARX_GIT_BRANCH")" --track "$SEARX_GIT_BRANCH" +git pull "$SEARX_GIT_BRANCH" +${SEARX_SRC}/manage.sh update_packages +EOF + configure_searx + + rst_title "${SEARX_SETTINGS}" section + rstBlock 'Diff between new setting file (<) and backup (>):' + echo + diff "$SEARX_SETTINGS}" "${SEARX_SETTINGS}.backup" + + local action + choose_one action "What should happen to the settings file? " \ + "keep new configuration" \ + "revert to the old configuration (backup file)" \ + "start interactiv shell" + case $action in + "keep new configuration") + info_msg "continue using new settings file" + ;; + "revert to the old configuration (backup file)") + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +cp -f ${SEARX_SETTINGS}.backup ${SEARX_SETTINGS} +EOF + ;; + "start interactiv shell") + interactive_shell + ;; + esac + chown "${SERVICE_USER}:${SERVICE_USER}" "${SEARX_SETTINGS}" + + # shellcheck disable=SC2016 + rst_para 'Diff between local modified settings (<) and $SEARX_GIT_BRANCH branch (>):' + echo + git_diff + wait_key + uWSGI_restart +} + +remove_all() { + rst_title "De-Install $SERVICE_NAME (service)" + remove_service + wait_key + remove_user +} + +assert_user() { + rst_title "user $SERVICE_USER" section + echo + tee_stderr 1 <<EOF | bash | prefix_stdout +sudo -H adduser --shell /bin/bash --system --home "$SERVICE_HOME" \ + --disabled-password --group --gecos 'searx' $SERVICE_USER +sudo -H usermod -a -G shadow $SERVICE_USER +groups $SERVICE_USER +EOF + #SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)" + #export SERVICE_HOME + #echo "export SERVICE_HOME=$SERVICE_HOME" +} + +remove_user() { + rst_title "Drop $SERVICE_USER HOME" section + if ask_yn "Do you really want to drop $SERVICE_USER home folder?"; then + userdel -r -f "$SERVICE_USER" 2>&1 | prefix_stdout + else + rst_para "Leave HOME folder $(du -sh "$SERVICE_HOME") unchanged." + fi +} + +# shellcheck disable=SC2164 +clone_searx(){ + rst_title "Clone searx sources" section + echo + git_clone "$SEARX_GIT_URL" "$SEARX_SRC" \ + "$SEARX_GIT_BRANCH" "$SERVICE_USER" + + pushd "${SEARX_SRC}" > /dev/null + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +cd "${SEARX_SRC}" +git config user.email "$ADMIN_EMAIL" +git config user.name "$ADMIN_NAME" +git checkout "$SEARX_GIT_BRANCH" +EOF + popd > /dev/null +} + +create_venv(){ + rst_title "Create virtualenv (python)" section + + rst_para "Create venv in ${SEARX_VENV} and install needed python packages." + echo + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +rm -rf "${SEARX_VENV}" +python3 -m venv "${SEARX_VENV}" +. ${SEARX_VENV}/bin/activate +${SEARX_SRC}/manage.sh update_packages +EOF + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +grep -qFs -- 'source ${SEARX_VENV}/bin/activate' ~/.profile \ + || echo 'source ${SEARX_VENV}/bin/activate' >> ~/.profile +EOF + +} + +configure_searx(){ + rst_title "Configure searx" section + rst_para "Setup searx config located at $SEARX_SETTINGS" + echo + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +sed -i -e "s/ultrasecretkey/$(openssl rand -hex 16)/g" "$SEARX_SETTINGS" +sed -i -e "s/{instance_name}/${SEARX_INSTANCE_NAME}/g" "$SEARX_SETTINGS" +EOF +} + +test_local_searx(){ + rstHeading "Testing searx instance localy" section + echo + tee_stderr 0.1 <<EOF | sudo -H -u "${SERVICE_USER}" -i 2>&1 | prefix_stdout "$_service_prefix" +cd ${SEARX_SRC} +sed -i -e "s/debug : False/debug : True/g" "$SEARX_SETTINGS" +timeout 5 python3 searx/webapp.py & +sleep 1 +curl --location --verbose --head --insecure http://127.0.0.1:8888/ +sed -i -e "s/debug : True/debug : False/g" "$SEARX_SETTINGS" +EOF + waitKEY +} + +install_searx_uwsgi() { + rst_title "Install searx's uWSGI app (searx.ini)" section + echo + uWSGI_install_app "$SEARX_UWSGI_APP" +} + +remove_searx_uwsgi() { + rst_title "Remove searx's uWSGI app (searx.ini)" section + echo + uWSGI_remove_app "$SEARX_UWSGI_APP" +} + +activate_service () { + rst_title "Activate $SERVICE_NAME (service)" section + uWSGI_enable_app "$SEARX_UWSGI_APP" +} + +deactivate_service () { + rst_title "De-Activate $SERVICE_NAME (service)" section + uWSGI_disable_app "$SEARX_UWSGI_APP" +} + +interactive_shell(){ + echo "// exit with CTRL-D" + sudo -H -u "${SERVICE_USER}" -i +} + +git_diff(){ + sudo -H -u "${SERVICE_USER}" -i <<EOF +cd ${SEARX_REPO_FOLDER} +git --no-pager diff +EOF +} + +show_service () { + rst_title "service status & log" + echo + systemctl status uwsgi.service + echo + read -r -s -n1 -t 5 -p "// use CTRL-C to stop monitoring the log" + echo + while true; do + trap break 2 + journalctl -f -u uwsgi.service + done + return 0 +} + +# ---------------------------------------------------------------------------- +main "$@" +# ---------------------------------------------------------------------------- |