diff options
| -rw-r--r-- | docs/admin/filtron.rst | 192 | ||||
| -rwxr-xr-x | utils/filtron.sh | 7 | ||||
| -rwxr-xr-x | utils/morty.sh | 2 | ||||
| -rwxr-xr-x | utils/searx.sh | 6 | ||||
| -rw-r--r-- | utils/templates/etc/filtron/rules.json | 202 |
5 files changed, 206 insertions, 203 deletions
diff --git a/docs/admin/filtron.rst b/docs/admin/filtron.rst index 205269973..2bc663411 100644 --- a/docs/admin/filtron.rst +++ b/docs/admin/filtron.rst @@ -39,6 +39,10 @@ an installation and the maintenance of, use our script :ref:`filtron.sh`. Sample configuration of filtron =============================== +.. sidebar:: Tooling box + + - :origin:`/etc/filtron/rules.json <utils/templates/etc/filtron/rules.json>` + An example configuration can be find below. This configuration limits the access of: @@ -50,100 +54,100 @@ of: .. code:: json - [{ - "name":"search request", - "filters":[ - "Param:q", - "Path=^(/|/search)$" - ], - "interval":"<time-interval-in-sec (int)>", - "limit":"<max-request-number-in-interval (int)>", - "subrules":[ - { - "name":"roboagent limit", - "interval":"<time-interval-in-sec (int)>", - "limit":"<max-request-number-in-interval (int)>", - "filters":[ - "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)" - ], - "actions":[ - { - "name":"block", - "params":{ - "message":"Rate limit exceeded" - } - } - ] - }, - { - "name":"botlimit", - "limit":0, - "stop":true, - "filters":[ - "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)" - ], - "actions":[ - { - "name":"block", - "params":{ - "message":"Rate limit exceeded" - } - } - ] - }, - { - "name":"IP limit", - "interval":"<time-interval-in-sec (int)>", - "limit":"<max-request-number-in-interval (int)>", - "stop":true, - "aggregations":[ - "Header:X-Forwarded-For" - ], - "actions":[ - { - "name":"block", - "params":{ - "message":"Rate limit exceeded" - } - } - ] - }, - { - "name":"rss/json limit", - "interval":"<time-interval-in-sec (int)>", - "limit":"<max-request-number-in-interval (int)>", - "stop":true, - "filters":[ - "Param:format=(csv|json|rss)" - ], - "actions":[ - { - "name":"block", - "params":{ - "message":"Rate limit exceeded" - } - } - ] - }, - { - "name":"useragent limit", - "interval":"<time-interval-in-sec (int)>", - "limit":"<max-request-number-in-interval (int)>", - "aggregations":[ - "Header:User-Agent" - ], - "actions":[ - { - "name":"block", - "params":{ - "message":"Rate limit exceeded" - } - } - ] - } - ] - }] - + [ + { "name": "search request", + "filters": [ + "Param:q", + "Path=^(/|/search)$" + ], + "interval": "<time-interval-in-sec (int)>", + "limit": "<max-request-number-in-interval (int)>", + "subrules": [ + { + "name": "roboagent limit", + "interval": "<time-interval-in-sec (int)>", + "limit": "<max-request-number-in-interval (int)>", + "filters": [ + "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "botlimit", + "limit": 0, + "stop": true, + "filters": [ + "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "IP limit", + "interval": "<time-interval-in-sec (int)>", + "limit": "<max-request-number-in-interval (int)>", + "stop": true, + "aggregations": [ + "Header:X-Forwarded-For" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "rss/json limit", + "interval": "<time-interval-in-sec (int)>", + "limit": "<max-request-number-in-interval (int)>", + "stop": true, + "filters": [ + "Param:format=(csv|json|rss)" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + }, + { + "name": "useragent limit", + "interval": "<time-interval-in-sec (int)>", + "limit": "<max-request-number-in-interval (int)>", + "aggregations": [ + "Header:User-Agent" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } + } + ] + } + ] + } + ] Route request through filtron diff --git a/utils/filtron.sh b/utils/filtron.sh index 3c545ff14..c1e360067 100755 --- a/utils/filtron.sh +++ b/utils/filtron.sh @@ -94,8 +94,9 @@ If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/" FILTRON_API : ${FILTRON_API} FILTRON_LISTEN : ${FILTRON_LISTEN} FILTRON_TARGET : ${FILTRON_TARGET} + EOF - [ ! -z ${1+x} ] && echo -e "$1" + [ ! -z ${1+x} ] && err_msg "$1" } main() { @@ -105,7 +106,7 @@ main() { dpkg apt-get install git wget curl \ || exit - local _usage="ERROR: unknown or missing $1 command $2" + local _usage="unknown or missing $1 command $2" case $1 in --source-only) ;; @@ -170,7 +171,7 @@ main() { *) usage "$_usage"; exit 42;; esac ;; - *) usage "ERROR: unknown or missing command $1"; exit 42;; + *) usage "unknown or missing command $1"; exit 42;; esac } diff --git a/utils/morty.sh b/utils/morty.sh index ba4e7dd2e..6c63351cf 100755 --- a/utils/morty.sh +++ b/utils/morty.sh @@ -84,7 +84,7 @@ ${DOT_CONFIG#"$REPO_ROOT/"} file:: SERVICE_USER : ${SERVICE_USER} EOF info_searx - [ ! -z ${1+x} ] && echo -e "$1" + [ ! -z ${1+x} ] && err_msg "$1" } info_searx() { diff --git a/utils/searx.sh b/utils/searx.sh index 707009a11..7609a4acf 100755 --- a/utils/searx.sh +++ b/utils/searx.sh @@ -110,7 +110,7 @@ If needed, set PUBLIC_URL of your WEB service in the '${DOT_CONFIG#"$REPO_ROOT/" SERVICE_USER : ${SERVICE_USER} EOF - [ ! -z ${1+x} ] && echo -e "$1" + [ ! -z ${1+x} ] && err_msg "$1" } main() { @@ -120,7 +120,7 @@ main() { dpkg systemctl apt-get install git wget curl \ || exit - local _usage="ERROR: unknown or missing $1 command $2" + local _usage="unknown or missing $1 command $2" case $1 in --source-only) ;; @@ -190,7 +190,7 @@ main() { *) usage "$_usage"; exit 42;; esac ;; - *) usage "ERROR: unknown or missing command $1"; exit 42;; + *) usage "unknown or missing command $1"; exit 42;; esac } diff --git a/utils/templates/etc/filtron/rules.json b/utils/templates/etc/filtron/rules.json index f333306e1..1c7005ae5 100644 --- a/utils/templates/etc/filtron/rules.json +++ b/utils/templates/etc/filtron/rules.json @@ -1,107 +1,105 @@ -[{ - "name":"search request", - "filters":[ - "Param:q", - "Path=^(/|/search)$" - ], - "interval":120, - "limit":19, - "actions":[ - { - "name":"block", - "params":{ - "message":"common rate limit exceeded" - } - } - ], - "subrules":[ - { - "name":"roboagent limit", - "interval":60, - "limit":3, - "filters":[ - "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client|Ruby)" - ], - "actions":[ - {"name":"log"}, - { - "name":"block", - "params":{ - "message":"rate limit exceeded" +[ + { "name": "suspiciously frequent IP", + "filters": [], + "interval": 600, + "limit": 30, + "aggregations": [ + "Header:X-Forwarded-For" + ], + "actions":[ + {"name":"log"} + ] + }, + { "name": "search request", + "filters": [ + "Param:q", + "Path=^(/|/search)$" + ], + "interval": 61, + "limit": 999, + "subrules": [ + { + "name": "roboagent limit", + "interval": 61, + "limit": 1, + "filters": [ + "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } } - } - ] - }, - { - "name":"botlimit", - "interval":60, - "limit":0, - "stop":true, - "filters":[ - "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)" - ], - "actions":[ - {"name":"log"}, - { - "name":"block", - "params":{ - "message":"rate limit exceeded" + ] + }, + { + "name": "botlimit", + "limit": 0, + "stop": true, + "filters": [ + "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } } - } - ] - }, - { - "name":"IP limit", - "interval":60, - "limit":13, - "stop":true, - "aggregations":[ - "Header:X-Forwarded-For" - ], - "actions":[ - {"name":"log"}, - { - "name":"block", - "params":{ - "message":"rate limit exceeded" + ] + }, + { + "name": "IP limit", + "interval": 61, + "limit": 9, + "stop": true, + "aggregations": [ + "Header:X-Forwarded-For" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } } - } - ] - }, - { - "name":"rss/json limit", - "interval":60, - "limit":13, - "stop":true, - "filters":[ - "Param:format=(csv|json|rss)" - ], - "actions":[ - {"name":"log"}, - { - "name":"block", - "params":{ - "message":"rate limit exceeded" + ] + }, + { + "name": "rss/json limit", + "interval": 121, + "limit": 2, + "stop": true, + "filters": [ + "Param:format=(csv|json|rss)" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } } - } - ] - }, - { - "name":"useragent limit", - "interval":60, - "limit":13, - "aggregations":[ - "Header:User-Agent" - ], - "actions":[ - {"name":"log"}, - { - "name":"block", - "params":{ - "message":"rate limit exceeded" + ] + }, + { + "name": "useragent limit", + "interval": 61, + "limit": 199, + "aggregations": [ + "Header:User-Agent" + ], + "actions": [ + { "name": "log"}, + { "name": "block", + "params": { + "message": "Rate limit exceeded" + } } - } - ] - } - ] -}] + ] + } + ] + } +] |