diff options
| author | Alexandre Flament <alex@al-f.net> | 2022-12-26 20:50:09 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2022-12-26 20:50:09 +0100 |
| commit | 0d4896170f5ea16d23c7acfa271d989c1ce53a8d (patch) | |
| tree | c3face46ae5cfd9a0dc3eefd4f12a4677f15b478 | |
| parent | e8f72d705209bee09738ba050aeb75927d663c84 (diff) | |
| parent | f3515041af527741cbdc383434d099c38e40dfba (diff) | |
| download | searxng-0d4896170f5ea16d23c7acfa271d989c1ce53a8d.tar.gz searxng-0d4896170f5ea16d23c7acfa271d989c1ce53a8d.zip | |
Merge pull request #2047 from dalf/fasttext-predict
Use fasttext-predict instead of fasttext(-wheel)
| -rw-r--r-- | Dockerfile | 3 | ||||
| -rw-r--r-- | requirements.txt | 2 | ||||
| -rw-r--r-- | searx/utils.py | 12 |
3 files changed, 7 insertions, 10 deletions
diff --git a/Dockerfile b/Dockerfile index ece20c86b..66f58395d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,7 +36,6 @@ RUN apk add --no-cache -t build-dependencies \ su-exec \ python3 \ py3-pip \ - py3-numpy \ libxml2 \ libxslt \ openssl \ @@ -44,8 +43,6 @@ RUN apk add --no-cache -t build-dependencies \ uwsgi \ uwsgi-python3 \ brotli \ - && pip3 install --no-cache setuptools wheel \ - && sed -i s/fasttext-wheel/fasttext/ requirements.txt \ && pip3 install --no-cache -r requirements.txt \ && apk del build-dependencies \ && rm -rf /root/.cache diff --git a/requirements.txt b/requirements.txt index 5c45fff6b..ea2616e1a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,4 +15,4 @@ setproctitle==1.3.2 redis==4.4.0 markdown-it-py==2.1.0 typing_extensions==4.4.0 -fasttext-wheel==0.9.2 +fasttext-predict==0.9.2.1 diff --git a/searx/utils.py b/searx/utils.py index 2157a4ce0..cda336035 100644 --- a/searx/utils.py +++ b/searx/utils.py @@ -15,7 +15,6 @@ from os.path import splitext, join from random import choice from html.parser import HTMLParser from urllib.parse import urljoin, urlparse -import fasttext from lxml import html from lxml.etree import ElementBase, XPath, XPathError, XPathSyntaxError, _ElementStringResult, _ElementUnicodeResult @@ -51,12 +50,9 @@ _STORAGE_UNIT_VALUE: Dict[str, int] = { _XPATH_CACHE: Dict[str, XPath] = {} _LANG_TO_LC_CACHE: Dict[str, Dict[str, str]] = {} -_FASTTEXT_MODEL: Optional[fasttext.FastText._FastText] = None +_FASTTEXT_MODEL: Optional["fasttext.FastText._FastText"] = None """fasttext model to predict laguage of a search term""" -# Monkey patch: prevent fasttext from showing a (useless) warning when loading a model. -fasttext.FastText.eprint = lambda x: None - class _NotSetClass: # pylint: disable=too-few-public-methods """Internal class for this module, do not create instance of this class. @@ -630,9 +626,13 @@ def eval_xpath_getindex(elements: ElementBase, xpath_spec: XPathSpecType, index: return default -def _get_fasttext_model() -> fasttext.FastText._FastText: +def _get_fasttext_model() -> "fasttext.FastText._FastText": global _FASTTEXT_MODEL # pylint: disable=global-statement if _FASTTEXT_MODEL is None: + import fasttext # pylint: disable=import-outside-toplevel + + # Monkey patch: prevent fasttext from showing a (useless) warning when loading a model. + fasttext.FastText.eprint = lambda x: None _FASTTEXT_MODEL = fasttext.load_model(str(data_dir / 'lid.176.ftz')) return _FASTTEXT_MODEL |