summaryrefslogtreecommitdiff
path: root/utils
diff options
context:
space:
mode:
authorAlexandre Flament <alex@al-f.net>2024-02-16 20:46:18 +0000
committerMarkus Heiser <markus.heiser@darmarIT.de>2024-02-20 10:43:20 +0100
commited66ed758dbd1c926296aa227da8a82fff9166e1 (patch)
tree1e7f818ee5dab0d7c09bad09d3586f0f87231584 /utils
parent76845ea42c2c3484e30c118f07671b13ade07a29 (diff)
downloadsearxng-ed66ed758dbd1c926296aa227da8a82fff9166e1.tar.gz
searxng-ed66ed758dbd1c926296aa227da8a82fff9166e1.zip
[mod] reduce memory footprint by not calling babel.Locale.parse at runtime
babel.Locale.parse loads more than 60MB in RAM. The only purpose is to get: LOCALE_NAMES - searx.data.LOCALES["LOCALE_NAMES"] RTL_LOCALES - searx.data.LOCALES["RTL_LOCALES"] This commit calls babel.Locale.parse when the translations are update from weblate and stored in:: searx/data/locales.json This file can be build by:: ./manage data.locales By store these variables in searx.data when the translations are updated we save round about 65MB (usually 4 worker = 260MB of RAM saved. Suggested-by: https://github.com/searxng/searxng/discussions/2633#discussioncomment-8490494 Co-authored-by: Markus Heiser <markus.heiser@darmarit.de>
Diffstat (limited to 'utils')
-rwxr-xr-xutils/lib_sxng_data.sh11
-rwxr-xr-xutils/lib_sxng_weblate.sh5
2 files changed, 16 insertions, 0 deletions
diff --git a/utils/lib_sxng_data.sh b/utils/lib_sxng_data.sh
index 549e6dbec..50a932f6d 100755
--- a/utils/lib_sxng_data.sh
+++ b/utils/lib_sxng_data.sh
@@ -7,6 +7,7 @@ data.:
all : update searx/sxng_locales.py and searx/data/*
traits : update searx/data/engine_traits.json & searx/sxng_locales.py
useragents: update searx/data/useragents.json with the most recent versions of Firefox
+ locales : update searx/data/locales.json from babel
EOF
}
@@ -16,6 +17,7 @@ data.all() {
pyenv.activate
data.traits
data.useragents
+ data.locales
build_msg DATA "update searx/data/osm_keys_tags.json"
pyenv.cmd python searxng_extra/update/update_osm_keys_tags.py
@@ -49,6 +51,15 @@ data.useragents() {
dump_return $?
}
+data.locales() {
+ ( set -e
+ pyenv.activate
+ build_msg DATA "update searx/data/locales.json"
+ python searxng_extra/update/update_locales.py
+ )
+ dump_return $?
+}
+
docs.prebuild() {
build_msg DOCS "build ${DOCS_BUILD}/includes"
(
diff --git a/utils/lib_sxng_weblate.sh b/utils/lib_sxng_weblate.sh
index f52b75d7c..f2b19257c 100755
--- a/utils/lib_sxng_weblate.sh
+++ b/utils/lib_sxng_weblate.sh
@@ -96,10 +96,15 @@ weblate.translations.commit() {
build_msg BABEL 'compile translation catalogs into binary MO files'
pybabel compile --statistics \
-d "searx/translations"
+
+ # update searx/data/translation_labels.json
+ data.locales
+
# git add/commit (no push)
commit_body=$(cd "${TRANSLATIONS_WORKTREE}"; git log --pretty=format:'%h - %as - %aN <%ae>' "${existing_commit_hash}..HEAD")
commit_message=$(echo -e "[translations] update from Weblate\n\n${commit_body}")
git add searx/translations
+ git add searx/data/locales.json
git commit -m "${commit_message}"
)
exitcode=$?