From aa887eb375224da7253f3d98d20b922705e28df6 Mon Sep 17 00:00:00 2001 From: Alexandre Flament Date: Tue, 19 Jan 2021 21:26:04 +0100 Subject: [mod] checker : replace pycld3 by langdetect pycld3 requires the native library cld3 langdetect is a pure python package --- searx/search/checker/impl.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'searx/search') diff --git a/searx/search/checker/impl.py b/searx/search/checker/impl.py index 244536f1b..25887b0f4 100644 --- a/searx/search/checker/impl.py +++ b/searx/search/checker/impl.py @@ -9,7 +9,8 @@ from time import time from urllib.parse import urlparse import re -import cld3 +from langdetect import detect_langs +from langdetect.lang_detect_exception import LangDetectException import requests.exceptions from searx import poolrequests, logger @@ -181,10 +182,14 @@ class ResultContainerTests: self.test_results.add_error(self.test_name, message, *args, '(' + sqstr + ')') def _add_language(self, text: str) -> typing.Optional[str]: - r = cld3.get_language(str(text)) # pylint: disable=E1101 - if r is not None and r.probability >= 0.98 and r.is_reliable: - self.languages.add(r.language) - self.test_results.add_language(r.language) + try: + r = detect_langs(str(text)) # pylint: disable=E1101 + except LangDetectException: + return None + + if len(r) > 0 and r[0].prob > 0.95: + self.languages.add(r[0].lang) + self.test_results.add_language(r[0].lang) return None def _check_result(self, result): -- cgit v1.2.3-54-g00ecf