diff options
| -rw-r--r-- | docs/admin/settings/settings_general.rst | 8 | ||||
| -rw-r--r-- | searx/metrics/__init__.py | 58 | ||||
| -rw-r--r-- | searx/openmetrics.py | 35 | ||||
| -rw-r--r-- | searx/settings.yml | 4 | ||||
| -rw-r--r-- | searx/settings_defaults.py | 1 | ||||
| -rwxr-xr-x | searx/webapp.py | 37 |
6 files changed, 130 insertions, 13 deletions
diff --git a/docs/admin/settings/settings_general.rst b/docs/admin/settings/settings_general.rst index 02a2156b3..75acb4f6d 100644 --- a/docs/admin/settings/settings_general.rst +++ b/docs/admin/settings/settings_general.rst @@ -13,6 +13,7 @@ donation_url: false contact_url: false enable_metrics: true + open_metrics: '' ``debug`` : ``$SEARXNG_DEBUG`` Allow a more detailed log if you run SearXNG directly. Display *detailed* error @@ -32,3 +33,10 @@ ``enable_metrics``: Enabled by default. Record various anonymous metrics available at ``/stats``, ``/stats/errors`` and ``/preferences``. + +``open_metrics``: + Disabled by default. Set to a secret password to expose an + `OpenMetrics API <https://github.com/prometheus/OpenMetrics>`_ at ``/metrics``, + e.g. for usage with Prometheus. The ``/metrics`` endpoint is using HTTP Basic Auth, + where the password is the value of ``open_metrics`` set above. The username used for + Basic Auth can be randomly chosen as only the password is being validated. diff --git a/searx/metrics/__init__.py b/searx/metrics/__init__.py index d7ccee91a..cc9b1b401 100644 --- a/searx/metrics/__init__.py +++ b/searx/metrics/__init__.py @@ -8,6 +8,7 @@ from timeit import default_timer from operator import itemgetter from searx.engines import engines +from searx.openmetrics import OpenMetricsFamily from .models import HistogramStorage, CounterStorage, VoidHistogram, VoidCounterStorage from .error_recorder import count_error, count_exception, errors_per_engines @@ -149,7 +150,9 @@ def get_reliabilities(engline_name_list, checker_results): checker_result = checker_results.get(engine_name, {}) checker_success = checker_result.get('success', True) errors = engine_errors.get(engine_name) or [] - if counter('engine', engine_name, 'search', 'count', 'sent') == 0: + sent_count = counter('engine', engine_name, 'search', 'count', 'sent') + + if sent_count == 0: # no request reliability = None elif checker_success and not errors: @@ -164,8 +167,9 @@ def get_reliabilities(engline_name_list, checker_results): reliabilities[engine_name] = { 'reliability': reliability, + 'sent_count': sent_count, 'errors': errors, - 'checker': checker_results.get(engine_name, {}).get('errors', {}), + 'checker': checker_result.get('errors', {}), } return reliabilities @@ -245,3 +249,53 @@ def get_engines_stats(engine_name_list): 'max_time': math.ceil(max_time_total or 0), 'max_result_count': math.ceil(max_result_count or 0), } + + +def openmetrics(engine_stats, engine_reliabilities): + metrics = [ + OpenMetricsFamily( + key="searxng_engines_response_time_total_seconds", + type_hint="gauge", + help_hint="The average total response time of the engine", + data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']], + data=[engine['total'] for engine in engine_stats['time']], + ), + OpenMetricsFamily( + key="searxng_engines_response_time_processing_seconds", + type_hint="gauge", + help_hint="The average processing response time of the engine", + data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']], + data=[engine['processing'] for engine in engine_stats['time']], + ), + OpenMetricsFamily( + key="searxng_engines_response_time_http_seconds", + type_hint="gauge", + help_hint="The average HTTP response time of the engine", + data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']], + data=[engine['http'] for engine in engine_stats['time']], + ), + OpenMetricsFamily( + key="searxng_engines_result_count_total", + type_hint="counter", + help_hint="The total amount of results returned by the engine", + data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']], + data=[engine['result_count'] for engine in engine_stats['time']], + ), + OpenMetricsFamily( + key="searxng_engines_request_count_total", + type_hint="counter", + help_hint="The total amount of user requests made to this engine", + data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']], + data=[engine_reliabilities.get(engine['name'], {}).get('sent_count', 0) for engine in engine_stats['time']], + ), + OpenMetricsFamily( + key="searxng_engines_reliability_total", + type_hint="counter", + help_hint="The overall reliability of the engine", + data_info=[{'engine_name': engine['name']} for engine in engine_stats['time']], + data=[ + engine_reliabilities.get(engine['name'], {}).get('reliability', 0) for engine in engine_stats['time'] + ], + ), + ] + return "".join([str(metric) for metric in metrics]) diff --git a/searx/openmetrics.py b/searx/openmetrics.py new file mode 100644 index 000000000..5232e05ca --- /dev/null +++ b/searx/openmetrics.py @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +"""Module providing support for displaying data in OpenMetrics format""" + + +class OpenMetricsFamily: # pylint: disable=too-few-public-methods + """A family of metrics. + The key parameter is the metric name that should be used (snake case). + The type_hint parameter must be one of 'counter', 'gauge', 'histogram', 'summary'. + The help_hint parameter is a short string explaining the metric. + The data_info parameter is a dictionary of descriptionary parameters for the data point (e.g. request method/path). + The data parameter is a flat list of the actual data in shape of a primive type. + + See https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md for more information. + """ + + def __init__(self, key: str, type_hint: str, help_hint: str, data_info: list, data: list): + self.key = key + self.type_hint = type_hint + self.help_hint = help_hint + self.data_info = data_info + self.data = data + + def __str__(self): + text_representation = f"""# HELP {self.key} {self.help_hint} +# TYPE {self.key} {self.type_hint} +""" + + for i, data_info_dict in enumerate(self.data_info): + if not data_info_dict and data_info_dict != 0: + continue + + info_representation = ','.join([f"{key}=\"{value}\"" for (key, value) in data_info_dict.items()]) + text_representation += f"{self.key}{{{info_representation}}} {self.data[i]}\n" + + return text_representation diff --git a/searx/settings.yml b/searx/settings.yml index d27172aef..27b78b8da 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -12,6 +12,10 @@ general: contact_url: false # record stats enable_metrics: true + # expose stats in open metrics format at /metrics + # leave empty to disable (no password set) + # open_metrics: <password> + open_metrics: '' brand: new_issue_url: https://github.com/searxng/searxng/issues/new diff --git a/searx/settings_defaults.py b/searx/settings_defaults.py index 06cae34bc..891cc1df3 100644 --- a/searx/settings_defaults.py +++ b/searx/settings_defaults.py @@ -143,6 +143,7 @@ SCHEMA = { 'contact_url': SettingsValue((None, False, str), None), 'donation_url': SettingsValue((bool, str), "https://docs.searxng.org/donate.html"), 'enable_metrics': SettingsValue(bool, True), + 'open_metrics': SettingsValue(str, ''), }, 'brand': { 'issue_url': SettingsValue(str, 'https://github.com/searxng/searxng/issues'), diff --git a/searx/webapp.py b/searx/webapp.py index d2d486d20..1c0158382 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -87,10 +87,7 @@ from searx.webadapter import ( get_selected_categories, parse_lang, ) -from searx.utils import ( - gen_useragent, - dict_subset, -) +from searx.utils import gen_useragent, dict_subset from searx.version import VERSION_STRING, GIT_URL, GIT_BRANCH from searx.query import RawTextQuery from searx.plugins import Plugin, plugins, initialize as plugin_initialize @@ -104,13 +101,7 @@ from searx.answerers import ( answerers, ask, ) -from searx.metrics import ( - get_engines_stats, - get_engine_errors, - get_reliabilities, - histogram, - counter, -) +from searx.metrics import get_engines_stats, get_engine_errors, get_reliabilities, histogram, counter, openmetrics from searx.flaskfix import patch_application from searx.locales import ( @@ -1218,6 +1209,30 @@ def stats_checker(): return jsonify(result) +@app.route('/metrics') +def stats_open_metrics(): + password = settings['general'].get("open_metrics") + + if not (settings['general'].get("enable_metrics") and password): + return Response('open metrics is disabled', status=404, mimetype='text/plain') + + if not request.authorization or request.authorization.password != password: + return Response('access forbidden', status=401, mimetype='text/plain') + + filtered_engines = dict(filter(lambda kv: request.preferences.validate_token(kv[1]), engines.items())) + + checker_results = checker_get_result() + checker_results = ( + checker_results['engines'] if checker_results['status'] == 'ok' and 'engines' in checker_results else {} + ) + + engine_stats = get_engines_stats(filtered_engines) + engine_reliabilities = get_reliabilities(filtered_engines, checker_results) + metrics_text = openmetrics(engine_stats, engine_reliabilities) + + return Response(metrics_text, mimetype='text/plain') + + @app.route('/robots.txt', methods=['GET']) def robots(): return Response( |