diff options
author | Alexandre Flament <alex@al-f.net> | 2024-02-16 20:46:18 +0000 |
---|---|---|
committer | Markus Heiser <markus.heiser@darmarIT.de> | 2024-02-20 10:43:20 +0100 |
commit | ed66ed758dbd1c926296aa227da8a82fff9166e1 (patch) | |
tree | 1e7f818ee5dab0d7c09bad09d3586f0f87231584 /utils | |
parent | 76845ea42c2c3484e30c118f07671b13ade07a29 (diff) | |
download | searxng-ed66ed758dbd1c926296aa227da8a82fff9166e1.tar.gz searxng-ed66ed758dbd1c926296aa227da8a82fff9166e1.zip |
[mod] reduce memory footprint by not calling babel.Locale.parse at runtime
babel.Locale.parse loads more than 60MB in RAM. The only purpose is to get:
LOCALE_NAMES - searx.data.LOCALES["LOCALE_NAMES"]
RTL_LOCALES - searx.data.LOCALES["RTL_LOCALES"]
This commit calls babel.Locale.parse when the translations are update from
weblate and stored in::
searx/data/locales.json
This file can be build by::
./manage data.locales
By store these variables in searx.data when the translations are updated we save
round about 65MB (usually 4 worker = 260MB of RAM saved.
Suggested-by: https://github.com/searxng/searxng/discussions/2633#discussioncomment-8490494
Co-authored-by: Markus Heiser <markus.heiser@darmarit.de>
Diffstat (limited to 'utils')
-rwxr-xr-x | utils/lib_sxng_data.sh | 11 | ||||
-rwxr-xr-x | utils/lib_sxng_weblate.sh | 5 |
2 files changed, 16 insertions, 0 deletions
diff --git a/utils/lib_sxng_data.sh b/utils/lib_sxng_data.sh index 549e6dbec..50a932f6d 100755 --- a/utils/lib_sxng_data.sh +++ b/utils/lib_sxng_data.sh @@ -7,6 +7,7 @@ data.: all : update searx/sxng_locales.py and searx/data/* traits : update searx/data/engine_traits.json & searx/sxng_locales.py useragents: update searx/data/useragents.json with the most recent versions of Firefox + locales : update searx/data/locales.json from babel EOF } @@ -16,6 +17,7 @@ data.all() { pyenv.activate data.traits data.useragents + data.locales build_msg DATA "update searx/data/osm_keys_tags.json" pyenv.cmd python searxng_extra/update/update_osm_keys_tags.py @@ -49,6 +51,15 @@ data.useragents() { dump_return $? } +data.locales() { + ( set -e + pyenv.activate + build_msg DATA "update searx/data/locales.json" + python searxng_extra/update/update_locales.py + ) + dump_return $? +} + docs.prebuild() { build_msg DOCS "build ${DOCS_BUILD}/includes" ( diff --git a/utils/lib_sxng_weblate.sh b/utils/lib_sxng_weblate.sh index f52b75d7c..f2b19257c 100755 --- a/utils/lib_sxng_weblate.sh +++ b/utils/lib_sxng_weblate.sh @@ -96,10 +96,15 @@ weblate.translations.commit() { build_msg BABEL 'compile translation catalogs into binary MO files' pybabel compile --statistics \ -d "searx/translations" + + # update searx/data/translation_labels.json + data.locales + # git add/commit (no push) commit_body=$(cd "${TRANSLATIONS_WORKTREE}"; git log --pretty=format:'%h - %as - %aN <%ae>' "${existing_commit_hash}..HEAD") commit_message=$(echo -e "[translations] update from Weblate\n\n${commit_body}") git add searx/translations + git add searx/data/locales.json git commit -m "${commit_message}" ) exitcode=$? |