diff options
| -rwxr-xr-x | utils/filtron.sh | 77 | ||||
| -rwxr-xr-x | utils/lib.sh | 72 | ||||
| -rw-r--r-- | utils/templates/etc/filtron/rules.json | 142 | ||||
| -rw-r--r-- | utils/templates/lib/systemd/system/filtron.service | 2 |
4 files changed, 207 insertions, 86 deletions
diff --git a/utils/filtron.sh b/utils/filtron.sh index f310991a0..593c0fcf7 100755 --- a/utils/filtron.sh +++ b/utils/filtron.sh @@ -11,6 +11,11 @@ source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" FILTRON_ETC="/etc/filtron" +FILTRON_RULES="$FILTRON_ETC/rules.json" +FILTRON_API="127.0.0.1:4005" +FILTRON_LISTEN="127.0.0.1:4004" +FILTRON_TARGET="127.0.0.1:8888" + SERVICE_NAME="filtron" SERVICE_USER="${SERVICE_NAME}" SERVICE_HOME="/home/${SERVICE_USER}" @@ -23,6 +28,11 @@ GO_ENV="${SERVICE_HOME}/.go_env" GO_PKG_URL="https://dl.google.com/go/go1.13.5.linux-amd64.tar.gz" GO_TAR=$(basename "$GO_PKG_URL") +CONFIG_FILES=( + "${FILTRON_RULES}" + "${SERVICE_SYSTEMD_UNIT}" +) + # ---------------------------------------------------------------------------- usage(){ # ---------------------------------------------------------------------------- @@ -37,10 +47,16 @@ usage: $(basename "$0") remove [all] $(basename "$0") activate [server] $(basename "$0") deactivate [server] - -shell - start interactive shell with user ${SERVICE_USER} -install user - add service user '$SERVICE_USER' at $SERVICE_HOME - + $(basename "$0") show [server] + +shell + start interactive shell from user ${SERVICE_USER} +show server + show server status and log +install / remove + all - complete setup of filtron server +install user + add service user '$SERVICE_USER' at $SERVICE_HOME EOF [ ! -z ${1+x} ] && echo -e "$1" } @@ -58,6 +74,14 @@ main(){ sudo_or_exit interactive_shell ;; + show) + case $2 in + server) + sudo_or_exit + show_server + ;; + *) usage "$_usage"; exit 42;; + esac ;; install) sudo_or_exit case $2 in @@ -91,21 +115,27 @@ main(){ install_all() { rst_title "Install $SERVICE_NAME (service)" assert_user + wait_key install_go + wait_key install_filtron + wait_key install_server + wait_key } remove_all() { rst_title "De-Install $SERVICE_NAME (service)" remove_server + wait_key remove_user - rm -rf "$FILTRON_ETC" + rm -r "$FILTRON_ETC" 2>&1 | prefix_stdout wait_key } install_server() { - rst_title "Install System-D Unit ${SERVICE_NAME}.service ..." section + rst_title "Install System-D Unit ${SERVICE_NAME}.service" section + echo install_template ${SERVICE_SYSTEMD_UNIT} root root 644 wait_key activate_server @@ -116,12 +146,12 @@ remove_server() { return fi deactivate_server - rm "${SERVICE_SYSTEMD_UNIT}" + rm "${SERVICE_SYSTEMD_UNIT}" 2>&1 | prefix_stdout } - activate_server () { rst_title "Activate $SERVICE_NAME (service)" section + echo tee_stderr <<EOF | bash 2>&1 | prefix_stdout systemctl enable $SERVICE_NAME.service systemctl restart $SERVICE_NAME.service @@ -129,7 +159,6 @@ EOF tee_stderr <<EOF | bash 2>&1 | prefix_stdout systemctl status $SERVICE_NAME.service EOF - wait_key } deactivate_server () { @@ -139,7 +168,6 @@ deactivate_server () { systemctl stop $SERVICE_NAME.service systemctl disable $SERVICE_NAME.service EOF - wait_key } assert_user() { @@ -168,18 +196,18 @@ EOF remove_user() { rst_title "Drop $SERVICE_USER HOME" section if ask_yn "Do you really want to drop $SERVICE_USER home folder?"; then - userdel -r -f "$SERVICE_USER" + userdel -r -f "$SERVICE_USER" 2>&1 | prefix_stdout else rst_para "Leave HOME folder $(du -sh "$SERVICE_HOME") unchanged." fi } interactive_shell(){ - echo "// exit with STRG-D" + echo "// exit with CTRL-D" sudo -H -u ${SERVICE_USER} -i } -_service_prefix="$SERVICE_USER@$(hostname) -->| " +_service_prefix=" |$SERVICE_USER| " install_go(){ rst_title "Install Go in user's HOME" section @@ -199,14 +227,29 @@ EOF ! which go >/dev/null && echo "Go Installation not found in PATH!?!" which go >/dev/null && go version && echo "congratulations -- Go installation OK :)" EOF - wait_key } install_filtron() { - tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" | prefix_stdout "$_service_prefix" -go get -v -u github.com/asciimoo/filtron 2>&1 + rst_title "Install filtron in user's ~/go-apps" section + echo + tee_stderr <<EOF | sudo -i -u "$SERVICE_USER" 2>&1 | prefix_stdout "$_service_prefix" +go get -v -u github.com/asciimoo/filtron EOF - install_template "$FILTRON_ETC/rules.json" root root 644 + install_template --no-eval "$FILTRON_RULES" root root 644 +} + +show_server () { + rst_title "server status & log" + echo + systemctl status filtron.service + echo + read -s -n1 -t 5 -p "// use CTRL-C to stop monitoring the log" + echo + while true; do + trap break 2 + journalctl -f -u filtron + done + return 0 } # ---------------------------------------------------------------------------- diff --git a/utils/lib.sh b/utils/lib.sh index b5e897549..d06cabf26 100755 --- a/utils/lib.sh +++ b/utils/lib.sh @@ -22,6 +22,13 @@ if [[ -z "$SYSTEMD_UNITS" ]]; then SYSTEMD_UNITS="/lib/systemd/system" fi +if [[ -z ${DIFF_CMD} ]]; then + DIFF_CMD="diff -u" + if command -v colordiff >/dev/null; then + DIFF_CMD="colordiff -u" + fi +fi + sudo_or_exit() { # usage: sudo_or_exit @@ -55,10 +62,10 @@ rst_para() { # usage: RST_INDENT=1 rst_para "lorem ipsum ..." local prefix='' if ! [[ -z $RST_INDENT ]] && [[ $RST_INDENT -gt 0 ]]; then - prefix="$(for i in $(seq 1 "$RST_INDENT"); do printf " "; done)" - echo -en "\n$*\n" | $FMT | prefix_stdout "$prefix" + prefix="$(for i in $(seq 1 "$RST_INDENT"); do printf " "; done)" + echo -en "\n$*\n" | $FMT | prefix_stdout "$prefix" else - echo -en "\n$*\n" | $FMT + echo -en "\n$*\n" | $FMT fi } @@ -66,15 +73,23 @@ err_msg() { echo -e "ERROR: $*" >&2; } warn_msg() { echo -e "WARN: $*" >&2; } info_msg() { echo -e "INFO: $*"; } +clean_stdin() { + if [[ $(uname -s) != 'Darwin' ]]; then + while $(read -n1 -t 0.1); do : ; done + fi +} + wait_key(){ # usage: waitKEY [<timeout in sec>] + clean_stdin local _t=$1 [[ ! -z $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT [[ ! -z $_t ]] && _t="-t $_t" # shellcheck disable=SC2086 - read -n1 $_t -p "** press any [KEY] to continue **" + read -s -n1 $_t -p "** press any [KEY] to continue **" echo + clean_stdin } ask_yn() { @@ -100,6 +115,7 @@ ask_yn() { esac echo while true; do + clean_stdin printf "$1 ${choice} " # shellcheck disable=SC2086 read -n1 $_t @@ -117,6 +133,7 @@ ask_yn() { _t="" err_msg "invalid choice" done + clean_stdin return $exit_val } @@ -144,7 +161,7 @@ tee_stderr () { prefix_stdout () { # usage: <cmd> | prefix_stdout [prefix] - local prefix="-->| " + local prefix=" | " if [[ ! -z $1 ]] ; then prefix="$1"; fi @@ -223,6 +240,7 @@ choose_one() { fi done while true; do + clean_stdin printf "$1 [$default] " if (( 10 > $max )); then @@ -242,6 +260,7 @@ choose_one() { err_msg "invalid choice" done echo + clean_stdin eval "$env_name"='${list[${REPLY}]}' } @@ -288,31 +307,48 @@ install_template() { mkdir -p "$(dirname "${dst}")" - if [[ -f "${dst}" ]] ; then - info_msg "file ${dst} allready exists on this host" + if [[ ! -f "${dst}" ]]; then + info_msg "install: ${template_file}" + sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \ + "${template_file}" "${dst}" | prefix_stdout + return $? + fi + + if [[ -f "${dst}" ]] && cmp --silent "${template_file}" "${dst}" ; then + info_msg "file ${dst} allready installed" + return 0 + fi + + info_msg "file ${dst} allready exists on this host" + + while true; do choose_one _reply "choose next step with file $dst" \ "replace file" \ - "leave file unchanged" + "leave file unchanged" \ + "interactiv shell" \ + "diff files" case $_reply in "replace file") info_msg "install: ${template_file}" sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \ "${template_file}" "${dst}" | prefix_stdout + break ;; "leave file unchanged") + break ;; "interactiv shell") - echo "// exit with STRG-D" + echo "// edit ${dst} to your needs" + echo "// exit with CTRL-D" sudo -H -u "${owner}" -i - ;; + $DIFF_CMD "${dst}" "${template_file}" + if ask_yn "did you edit ${template_file} to your needs?"; then + break + fi + ;; + "diff files") + $DIFF_CMD "${dst}" "${template_file}" | prefix_stdout esac - - else - info_msg "install: ${template_file}" - sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \ - "${template_file}" "${dst}" | prefix_stdout - fi - + done } - diff --git a/utils/templates/etc/filtron/rules.json b/utils/templates/etc/filtron/rules.json index 4a232388e..b54e097a5 100644 --- a/utils/templates/etc/filtron/rules.json +++ b/utils/templates/etc/filtron/rules.json @@ -1,56 +1,98 @@ -[ +[{ + "name":"search request", + "filters":[ + "Param:q", + "Path=^(/|/search)$" + ], + "interval":60, + "limit":15, + "subrules":[ { - "name": "api limit", - "interval": 60, - "limit": 1000, - "filters": ["Path=^/api"], - "aggregations": ["Path"], - "actions": [ - {"name": "block"} - ], - "subrules": [ - { - "name": "drop put", - "interval": 60, - "limit": 100, - "filters": ["Method=PUT"], - "aggregations": ["Header:X-Forwarded-For"], - "actions": [ - {"name": "shell", - "params": {"cmd": "iptables -A INPUT -s %v -j DROP", "args": ["Header:X-Forwarded-For"]}} - ] - } - ] + "name":"roboagent limit", + "interval":60, + "limit":15, + "filters":[ + "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)" + ], + "actions":[ + {"name": "log"}, + { + "name":"block", + "params":{ + "message":"Rate limit exceeded" + } + } + ] }, { - "name": "log'n'block rss", - "interval": 300, - "limit": 2500, - "filters": ["Path=^/$", "GET:format=rss"], - "actions": [ - {"name": "log"}, - {"name": "block"} - ] + "name":"botlimit", + "limit":0, + "stop":true, + "filters":[ + "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)" + ], + "actions":[ + {"name": "log"}, + { + "name":"block", + "params":{ + "message":"Rate limit exceeded" + } + } + ] }, { - "name": "log rule", - "filters": ["Path=/"], - "actions": [ {"name": "log"} ], - "subrules": [ - { - "name": "block missing accept-language", - "filters": ["!Header:Accept-Language"], - "actions": [ - {"name": "block"} - ] - }, - { - "name": "block curl", - "filters": ["Header:User-Agent=[Cc]url"], - "actions": [ - {"name": "block"} - ] - } - ] + "name":"IP limit", + "interval":60, + "limit":15, + "stop":true, + "aggregations":[ + "Header:X-Forwarded-For" + ], + "actions":[ + {"name": "log"}, + { + "name":"block", + "params":{ + "message":"Rate limit exceeded" + } + } + ] + }, + { + "name":"rss/json limit", + "interval":60, + "limit":15, + "stop":true, + "filters":[ + "Param:format=(csv|json|rss)" + ], + "actions":[ + {"name": "log"}, + { + "name":"block", + "params":{ + "message":"Rate limit exceeded" + } + } + ] + }, + { + "name":"useragent limit", + "interval":60, + "limit":15, + "aggregations":[ + "Header:User-Agent" + ], + "actions":[ + {"name": "log"}, + { + "name":"block", + "params":{ + "message":"Rate limit exceeded" + } + } + ] } -] + ] +}] diff --git a/utils/templates/lib/systemd/system/filtron.service b/utils/templates/lib/systemd/system/filtron.service index fdb67731a..3b0c6edcc 100644 --- a/utils/templates/lib/systemd/system/filtron.service +++ b/utils/templates/lib/systemd/system/filtron.service @@ -10,7 +10,7 @@ Type=simple User=${SERVICE_USER} Group=${SERVICE_GROUP} WorkingDirectory=${SERVICE_HOME} -ExecStart=${SERVICE_HOME}/go-apps/bin/filtron -rules ${FILTRON_RULES} +ExecStart=${SERVICE_HOME}/go-apps/bin/filtron -api '${FILTRON_API}' -listen '${FILTRON_LISTEN}' -rules '${FILTRON_RULES}' -target '${FILTRON_TARGET}' Restart=always Environment=USER=${SERVICE_USER} HOME=${SERVICE_HOME} |