diff options
33 files changed, 163 insertions, 125 deletions
diff --git a/.config.sh b/.config.sh index f9bac7383..7eed5f681 100644 --- a/.config.sh +++ b/.config.sh @@ -28,7 +28,10 @@ fi # SEARX_INTERNAL_URL="127.0.0.1:8888" # SEARX_SETTINGS_TEMPLATE="${REPO_ROOT}/utils/templates/etc/searx/use_default_settings.yml" -# Only change, if you maintain a searx brand in your searx fork. +# Only change, if you maintain a searx brand in your searx fork (GIT_URL) which +# is not hold by branch 'master'. The branch has to be a local branch, in the +# repository from which you install (which is most often the case). If you want +# to install branch 'foo', don't forget to run 'git branch foo origin/foo' once. # GIT_BRANCH="${GIT_BRANCH:-master}" # filtron.sh diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index 67627835d..70e490c0d 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -6,6 +6,8 @@ labels: bug assignees: '' --- +<!-- PLEASE FILL THESE FIELDS, IT REALLY HELPS THE MAINTAINERS OF SEARX --> + **Version of Searx, commit number if you are using on master branch and stipulate if you forked Searx** <!-- If you are running on master branch using git execute this command in order to fetch the latest commit ID: @@ -34,4 +36,4 @@ or manually by executing the searx/webapp.py file? --> <!-- If applicable, add screenshots, logs to help explain your problem. --> **Additional context** -<!-- Add any other context about the problem here. -->
\ No newline at end of file +<!-- Add any other context about the problem here. --> diff --git a/.github/ISSUE_TEMPLATE/engine-request.md b/.github/ISSUE_TEMPLATE/engine-request.md index 1037004d6..3074316a7 100644 --- a/.github/ISSUE_TEMPLATE/engine-request.md +++ b/.github/ISSUE_TEMPLATE/engine-request.md @@ -6,6 +6,8 @@ labels: enhancement, engine request assignees: '' --- +<!-- PLEASE FILL THESE FIELDS, IT REALLY HELPS THE MAINTAINERS OF SEARX --> + **Working URL to the engine** <!-- Please check if the engine is responding correctly before submitting it. --> @@ -26,4 +28,4 @@ general, files, images, it, map, music, news, science, social media and videos. You can add multiple categories at the same time. --> **Additional context** -<!-- Add any other context about this engine here. -->
\ No newline at end of file +<!-- Add any other context about this engine here. --> diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md index 3bb27fefd..63e24b85b 100644 --- a/.github/ISSUE_TEMPLATE/feature-request.md +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -6,6 +6,8 @@ labels: enhancement assignees: '' --- +<!-- PLEASE FILL THESE FIELDS, IT REALLY HELPS THE MAINTAINERS OF SEARX --> + **Is your feature request related to a problem? Please describe.** <!-- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] --> @@ -16,4 +18,4 @@ assignees: '' <!-- A clear and concise description of any alternative solutions or features you've considered. --> **Additional context** -<!-- Add any other context or screenshots about the feature request here. -->
\ No newline at end of file +<!-- Add any other context or screenshots about the feature request here. --> diff --git a/docs/_themes/searx/static/searx.css b/docs/_themes/searx/static/searx.css index 0f2eff728..ae742a4e0 100644 --- a/docs/_themes/searx/static/searx.css +++ b/docs/_themes/searx/static/searx.css @@ -138,32 +138,3 @@ caption { caption-side: top; text-align: left; } - -/* bugs since sphinx 3.1 - -See sphinx-doc project, PR 7838 & 7484 with elementary patch to the basic CSS: - -- https://github.com/sphinx-doc/sphinx/issues/7838#issuecomment-646009605 -- https://github.com/sphinx-doc/sphinx/pull/7484#issuecomment-646058972 - -*/ - -li > p:first-child { - margin-top: 0; -} - -li > p:last-child { - margin-bottom: 0; -} - -div.admonition dl { - margin-bottom: 0; -} - -div.sidebar { - clear: none; -} - -div.admonition, div.topic, pre { - clear: none; -} diff --git a/docs/admin/engines.rst b/docs/admin/engines.rst index 0ec86a614..3ad206303 100644 --- a/docs/admin/engines.rst +++ b/docs/admin/engines.rst @@ -46,7 +46,7 @@ Show errors **DE** .. _configured engines: -.. jinja:: webapp +.. jinja:: searx .. flat-table:: Engines configured at built time (defaults) :header-rows: 1 diff --git a/docs/admin/plugins.rst b/docs/admin/plugins.rst index 4ed9066fd..d97b3dada 100644 --- a/docs/admin/plugins.rst +++ b/docs/admin/plugins.rst @@ -14,7 +14,7 @@ Configuration defaults (at built time): .. _configured plugins: -.. jinja:: webapp +.. jinja:: searx .. flat-table:: Plugins configured at built time (defaults) :header-rows: 1 diff --git a/docs/conf.py b/docs/conf.py index d6fde9bec..0c07761a8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -27,9 +27,15 @@ numfig = True exclude_patterns = ['build-templates/*.rst'] -from searx import webapp +import searx.search +import searx.engines +import searx.plugins +searx.search.initialize() jinja_contexts = { - 'webapp': dict(**webapp.__dict__), + 'searx': { + 'engines': searx.engines.engines, + 'plugins': searx.plugins.plugins + }, } # usage:: lorem :patch:`f373169` ipsum diff --git a/docs/dev/makefile.rst b/docs/dev/makefile.rst index 699729a28..c43855617 100644 --- a/docs/dev/makefile.rst +++ b/docs/dev/makefile.rst @@ -150,6 +150,35 @@ Documentation <contrib docs>` section. If you want to edit the documentation read our :ref:`make docs-live` section. If you are working in your own brand, adjust your :ref:`Makefile setup <makefile setup>`. +.. _make books: + +``make books/{name}.html books/{name}.pdf`` +=========================================== + +.. _intersphinx: https://www.sphinx-doc.org/en/stable/ext/intersphinx.html +.. _XeTeX: https://tug.org/xetex/ + +.. sidebar:: info + + To build PDF a XeTeX_ is needed, see :ref:`buildhosts`. + + +The ``books/{name}.*`` targets are building *books*. A *book* is a +sub-directory containing a ``conf.py`` file. One example is the user handbook +which can deployed separately (:origin:`docs/user/conf.py`). Such ``conf.py`` +do inherit from :origin:`docs/conf.py` and overwrite values to fit *book's* +needs. + +With the help of Intersphinx_ (:ref:`reST smart ref`) the links to searx’s +documentation outside of the book will be bound by the object inventory of +``DOCS_URL``. Take into account that URLs will be picked from the inventary at +documentation's build time. + +Use ``make docs-help`` to see which books available: + +.. program-output:: bash -c "cd ..; make --no-print-directory docs-help" + :ellipsis: 0,-6 + .. _make gh-pages: diff --git a/docs/dev/reST.rst b/docs/dev/reST.rst index 963378748..39cd9f671 100644 --- a/docs/dev/reST.rst +++ b/docs/dev/reST.rst @@ -319,6 +319,9 @@ To list all anchors of the inventory (e.g. ``python``) use: .. code:: sh $ python -m sphinx.ext.intersphinx https://docs.python.org/3/objects.inv + ... + $ python -m sphinx.ext.intersphinx https://searx.github.io/searx/objects.inv + ... Literal blocks ============== @@ -1286,15 +1289,21 @@ build chapter: :ref:`engines generic`. Below the jinja directive from the :language: reST :start-after: .. _configured engines: -The context for the template is selected in the line ``.. jinja:: webapp``. In -sphinx's build configuration (:origin:`docs/conf.py`) the ``webapp`` context -points to the name space of the python module: ``webapp``. +The context for the template is selected in the line ``.. jinja:: searx``. In +sphinx's build configuration (:origin:`docs/conf.py`) the ``searx`` context +contains the ``engines`` and ``plugins``. .. code:: py - from searx import webapp + import searx.search + import searx.engines + import searx.plugins + searx.search.initialize() jinja_contexts = { - 'webapp': dict(**webapp.__dict__) + 'searx': { + 'engines': searx.engines.engines, + 'plugins': searx.plugins.plugins + }, } diff --git a/docs/dev/search_api.rst b/docs/dev/search_api.rst index b63891f53..68fee94bf 100644 --- a/docs/dev/search_api.rst +++ b/docs/dev/search_api.rst @@ -6,7 +6,7 @@ Search API The search supports both ``GET`` and ``POST``. -Furthermore, two enpoints ``/`` and ``/search`` are available for querying. +Furthermore, two endpoints ``/`` and ``/search`` are available for querying. ``GET /`` diff --git a/docs/user/conf.py b/docs/user/conf.py index 53ade4b63..da2e27533 100644 --- a/docs/user/conf.py +++ b/docs/user/conf.py @@ -4,6 +4,8 @@ project = 'Searx User-HB' version = release = VERSION_STRING +intersphinx_mapping['searx'] = (DOCS_URL, None) + # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). diff --git a/docs/user/own-instance.rst b/docs/user/own-instance.rst index af415b61d..bb4c5d524 100644 --- a/docs/user/own-instance.rst +++ b/docs/user/own-instance.rst @@ -56,9 +56,9 @@ results. I see. What about private instances? ------------------------------------ -If users run their own instances, everything is in their control: the source -code, logging settings and private data. Unknown instance administrators do not -have to be trusted. +If users run their :ref:`own instances <installation>`, everything is in their +control: the source code, logging settings and private data. Unknown instance +administrators do not have to be trusted. Furthermore, as the default settings of their instance is editable, there is no need to use cookies to tailor searx to their needs. So preferences will not be @@ -38,7 +38,7 @@ install_geckodriver() { if [ -z "$NOTFOUND" ]; then return fi - GECKODRIVER_VERSION="v0.24.0" + GECKODRIVER_VERSION="v0.28.0" PLATFORM="`python3 -c "import platform; print(platform.system().lower(), platform.architecture()[0])"`" case "$PLATFORM" in "linux 32bit" | "linux2 32bit") ARCH="linux32";; diff --git a/requirements-dev.txt b/requirements-dev.txt index d0d1ecc95..ed3c1c035 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,18 +1,17 @@ -mock==2.0.0 +mock==4.0.3; python_version >= "3.6" +mock==2.0.0; python_version < "3.6" nose2[coverage_plugin]==0.9.2 cov-core==1.15.0 pycodestyle==2.6.0 -pylint==2.4.4 -plone.testing==5.0.0 -splinter==0.11.0 -transifex-client==0.12.2 +pylint==2.6.0 +splinter==0.14.0 +transifex-client==0.14.2 unittest2==1.1.0 -zope.testrunner==4.5.1 selenium==3.141.0 -twine==3.2.0; python_version >= "3.6" +twine==3.3.0; python_version >= "3.6" twine==1.15.0; python_version < "3.6" Pallets-Sphinx-Themes==1.2.3 -Sphinx==3.2.1; python_version >= '3.6' +Sphinx==3.4.1; python_version >= '3.6' Sphinx==3.0.1; python_version < '3.6' sphinx-issues==1.2.0 sphinx-jinja==1.1.1 @@ -21,4 +20,4 @@ sphinx-tabs==1.1.13; python_version < '3.6' sphinxcontrib-programoutput==0.16 sphinx-autobuild==2020.9.1; python_version >= '3.6' sphinx-autobuild==0.7.1; python_version < '3.6' -linuxdoc @ git+http://github.com/return42/linuxdoc.git@70673dcf69e705e08d81f53794895dc15c4920b3#egg=linuxdoc +linuxdoc==20210110 diff --git a/requirements.txt b/requirements.txt index 51a4fff72..e0c48ca94 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,12 @@ -certifi==2020.6.20 -babel==2.7.0 -flask-babel==1.0.0 +certifi==2020.12.05 +babel==2.9.0 +flask-babel==2.0.0 flask==1.1.2 idna==2.10 -jinja2==2.11.1 -lxml==4.6.1 +jinja2==2.11.2 +lxml==4.6.2 pygments==2.1.3 -pyopenssl==19.1.0 -python-dateutil==2.8.0 +pyopenssl==20.0.1 +python-dateutil==2.8.1 pyyaml==5.3.1 -requests[socks]==2.24.0 +requests[socks]==2.25.1 diff --git a/searx/__init__.py b/searx/__init__.py index 9bbc7c8c3..08e67f69d 100644 --- a/searx/__init__.py +++ b/searx/__init__.py @@ -60,7 +60,3 @@ if 'SEARX_SECRET' in environ: settings['server']['secret_key'] = environ['SEARX_SECRET'] if 'SEARX_BIND_ADDRESS' in environ: settings['server']['bind_address'] = environ['SEARX_BIND_ADDRESS'] - -if not searx_debug and settings['server']['secret_key'] == 'ultrasecretkey': - logger.error('server.secret_key is not changed. Please use something else instead of ultrasecretkey.') - exit(1) diff --git a/searx/brand.py b/searx/brand.py index 0eaaf0be3..7fcab6fad 100644 --- a/searx/brand.py +++ b/searx/brand.py @@ -4,4 +4,4 @@ ISSUE_URL = 'https://github.com/searx/searx/issues' SEARX_URL = 'https://searx.me' DOCS_URL = 'https://searx.github.io/searx' PUBLIC_INSTANCES = 'https://searx.space' -CONTACT_URL = 'mailto:contact@example.com' +CONTACT_URL = '' diff --git a/searx/query.py b/searx/query.py index e61e24f2c..38cb03ffe 100644 --- a/searx/query.py +++ b/searx/query.py @@ -77,7 +77,7 @@ class RawTextQuery: pass # this force a language - if query_part[0] == ':': + if query_part[0] == ':' and len(query_part) > 1: lang = query_part[1:].lower().replace('_', '-') # check if any language-code is equal with diff --git a/searx/settings.yml b/searx/settings.yml index e263e3ad4..0a569e59c 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -492,7 +492,7 @@ engines: - name : library genesis engine : xpath search_url : http://libgen.rs/search.php?req={query} - url_xpath : //a[contains(@href,"bookfi.net")]/@href + url_xpath : //a[contains(@href,"bookfi.net/md5")]/@href title_xpath : //a[contains(@href,"book/")]/text()[1] content_xpath : //td/a[1][contains(@href,"=author")]/text() categories : general @@ -906,17 +906,6 @@ engines: # See : http://mymemory.translated.net/doc/usagelimits.php # api_key : '' - - name : voat - engine: xpath - shortcut: vo - categories: social media - search_url : https://searchvoat.co/?t={query} - url_xpath : //div[@class="entry"]//p[@class="title"]/a/@href - title_xpath : //div[@class="entry"]//p[@class="title"]/a/text() - content_xpath : //div[@class="entry"]//span[@class="domain"]/a/text() - timeout : 10.0 - disabled : True - - name : 1337x engine : 1337x shortcut : 1337x diff --git a/searx/templates/oscar/macros.html b/searx/templates/oscar/macros.html index f40eebd37..571242860 100644 --- a/searx/templates/oscar/macros.html +++ b/searx/templates/oscar/macros.html @@ -14,7 +14,7 @@ <!-- Draw result header --> {% macro result_header(result, favicons, id) -%} -<h4 class="result_header" id="result-{{id}}">{% if result.engine~".png" in favicons %}{{ draw_favicon(result.engine) }} {% endif %}{% if result.url %}{{ result_link(result.url, result.title|safe, id=id) }}{% else %}{{ result.title|safe}}{% endif %}</h4> +<h4 class="result_header" id="result-{{id}}">{% if result.engine~".png" in favicons %}{{ draw_favicon(result.engine) }}{% endif %}{% if result.url %}{{ result_link(result.url, result.title|safe, id=id) }}{% else %}{{ result.title|safe}}{% endif %}</h4> {%- endmacro %} <!-- Draw result sub header --> diff --git a/searx/webapp.py b/searx/webapp.py index 5076355d0..49750d210 100755 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -86,6 +86,11 @@ from searx.metrology.error_recorder import errors_per_engines from werkzeug.serving import WSGIRequestHandler WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server'].get('http_protocol_version', '1.0')) +# check secret_key +if not searx_debug and settings['server']['secret_key'] == 'ultrasecretkey': + logger.error('server.secret_key is not changed. Please use something else instead of ultrasecretkey.') + exit(1) + # about static static_path = get_resources_directory(searx_dir, 'static', settings['ui']['static_path']) logger.debug('static directory is %s', static_path) diff --git a/tests/unit/test_query.py b/tests/unit/test_query.py index eff4d643c..0176f735e 100644 --- a/tests/unit/test_query.py +++ b/tests/unit/test_query.py @@ -59,6 +59,15 @@ class TestQuery(SearxTestCase): self.assertEqual(len(query.languages), 0) self.assertFalse(query.specific) + def test_empty_colon_in_query(self): + query_text = 'the : query' + query = RawTextQuery(query_text, []) + + self.assertEqual(query.getFullQuery(), query_text) + self.assertEqual(len(query.query_parts), 0) + self.assertEqual(len(query.languages), 0) + self.assertFalse(query.specific) + def test_timeout_below100(self): query_text = '<3 the query' query = RawTextQuery(query_text, []) diff --git a/utils/brand.env b/utils/brand.env index 660160e43..2e763159d 100644 --- a/utils/brand.env +++ b/utils/brand.env @@ -4,4 +4,4 @@ export ISSUE_URL='https://github.com/searx/searx/issues' export SEARX_URL='https://searx.me' export DOCS_URL='https://searx.github.io/searx' export PUBLIC_INSTANCES='https://searx.space' -export CONTACT_URL='mailto:contact@example.com' +export CONTACT_URL='' diff --git a/utils/lib.sh b/utils/lib.sh index 0d93138cc..8ae6bdd44 100755 --- a/utils/lib.sh +++ b/utils/lib.sh @@ -41,7 +41,7 @@ DOT_CONFIG="${DOT_CONFIG:-${REPO_ROOT}/.config.sh}" source_dot_config() { if [[ ! -e "${DOT_CONFIG}" ]]; then - err_msg "configuration does not extsts at: ${DOT_CONFIG}" + err_msg "configuration does not exists at: ${DOT_CONFIG}" return 42 fi # shellcheck disable=SC1090 @@ -159,7 +159,7 @@ clean_stdin() { } wait_key(){ - # usage: waitKEY [<timeout in sec>] + # usage: wait_key [<timeout in sec>] clean_stdin local _t=$1 @@ -639,7 +639,7 @@ nginx_distro_setup() { NGINX_DEFAULT_SERVER=/etc/nginx/nginx.conf # Including *location* directives from a dedicated config-folder into the - # server directive is, what what fedora and centos (already) does. + # server directive is, what fedora and centos (already) does. NGINX_APPS_ENABLED="/etc/nginx/default.d" # We add a apps-available folder and linking configurations into the @@ -692,7 +692,7 @@ nginx_reload() { nginx_install_app() { - # usage: nginx_install_app [<template option> ...] <myapp> + # usage: nginx_install_app [<template option> ...] <myapp.conf> # # <template option>: see install_template @@ -1381,7 +1381,7 @@ in_container() { LXC_ENV_FOLDER= if in_container; then # shellcheck disable=SC2034 - LXC_ENV_FOLDER="lxc/$(hostname)/" + LXC_ENV_FOLDER="lxc-env/$(hostname)/" fi lxc_init_container_env() { @@ -1413,8 +1413,8 @@ LXC_BASE_PACKAGES_centos="bash git python3" case $DIST_ID in ubuntu|debian) LXC_BASE_PACKAGES="${LXC_BASE_PACKAGES_debian}" ;; arch) LXC_BASE_PACKAGES="${LXC_BASE_PACKAGES_arch}" ;; - fedora) LXC_BASE_PACKAGES="${LXC_BASE_PACKAGES_fedora}" ;; - centos) LXC_BASE_PACKAGES="${LXC_BASE_PACKAGES_centos}" ;; + fedora) LXC_BASE_PACKAGES="${LXC_BASE_PACKAGES_fedora}" ;; + centos) LXC_BASE_PACKAGES="${LXC_BASE_PACKAGES_centos}" ;; *) err_msg "$DIST_ID-$DIST_VERS: pkg_install LXC_BASE_PACKAGES not yet implemented" ;; esac @@ -1429,9 +1429,9 @@ lxc_install_base_packages() { lxc_image_copy() { - # usage: lxc_copy_image <remote image> <local image> + # usage: lxc_image_copy <remote image> <local image> # - # lxc_copy_image "images:ubuntu/19.10" "ubu1910" + # lxc_image_copy "images:ubuntu/20.04" "ubu2004" if lxc_image_exists "local:${LXC_SUITE[i+1]}"; then info_msg "image ${LXC_SUITE[i]} already copied --> ${LXC_SUITE[i+1]}" diff --git a/utils/lxc-searx.env b/utils/lxc-searx.env index 3ce10021b..3121b63ce 100644 --- a/utils/lxc-searx.env +++ b/utils/lxc-searx.env @@ -23,8 +23,8 @@ lxc_set_suite_env() { # end of standard support see https://wiki.ubuntu.com/Releases "$LINUXCONTAINERS_ORG_NAME:ubuntu/16.04" "ubu1604" # April 2021 "$LINUXCONTAINERS_ORG_NAME:ubuntu/18.04" "ubu1804" # April 2023 - "$LINUXCONTAINERS_ORG_NAME:ubuntu/19.10" "ubu1910" # July 2020 - "$LINUXCONTAINERS_ORG_NAME:ubuntu/20.04" "ubu2004" # future (EOL 2030) + "$LINUXCONTAINERS_ORG_NAME:ubuntu/20.04" "ubu2004" # April 2025 + "$LINUXCONTAINERS_ORG_NAME:ubuntu/20.10" "ubu2010" # July 2021 # EOL see https://fedoraproject.org/wiki/Releases "$LINUXCONTAINERS_ORG_NAME:fedora/31" "fedora31" diff --git a/utils/lxc.sh b/utils/lxc.sh index 11dbe578d..39701ef04 100755 --- a/utils/lxc.sh +++ b/utils/lxc.sh @@ -32,14 +32,15 @@ apt-get install -y git curl wget " ubu1804_boilerplate="$ubu1604_boilerplate" ubu1904_boilerplate="$ubu1804_boilerplate" -ubu1910_boilerplate="$ubu1904_boilerplate" # shellcheck disable=SC2034 ubu2004_boilerplate=" -$ubu1910_boilerplate +$ubu1904_boilerplate echo 'Set disable_coredump false' >> /etc/sudo.conf " +ubu2010_boilerplate="$ubu1904_boilerplate" + # shellcheck disable=SC2034 archlinux_boilerplate=" pacman -Syu --noconfirm diff --git a/utils/makefile.include b/utils/makefile.include index 35a5134d9..879dcc23c 100644 --- a/utils/makefile.include +++ b/utils/makefile.include @@ -5,7 +5,7 @@ PHONY += lxc-activate lxc-purge lxc-activate: @$(MAKE) -s -f "$$(dirname $(abspath $(lastword $(MAKEFILE_LIST))))/makefile.lxc" lxc-activate lxc-purge: - $(Q)rm -rf ./lxc + $(Q)rm -rf ./lxc-env else include /.lxcenv.mk endif diff --git a/utils/makefile.lxc b/utils/makefile.lxc index fd999e40d..95a8d4317 100644 --- a/utils/makefile.lxc +++ b/utils/makefile.lxc @@ -9,10 +9,10 @@ # sudo make ./utils/makefile.lxc lxc-activate # sudo make ./utils/makefile.lxc lxc-deactivate -LXC_ENV_FOLDER=lxc/$(shell hostname)/ +LXC_ENV_FOLDER=lxc-env/$(shell hostname)/ lxc-help:: - @echo 'LXC: running in container LXC_ENV_FOLDER=$(LXC_ENV_FOLDER)' + @echo ' LXC: running in container LXC_ENV_FOLDER=$(LXC_ENV_FOLDER)' # If not activated, serve target 'lxc-activate' .. ifeq (,$(wildcard /.lxcenv.mk)) @@ -24,6 +24,9 @@ else PHONY += lxc-deactivate lxc-deactivate: rm /.lxcenv.mk +$(LXC_ENV_FOLDER): + $(Q)mkdir -p $(LXC_ENV_FOLDER) + $(Q)echo placeholder > $(LXC_ENV_FOLDER).placeholder endif .PHONY: $(PHONY) diff --git a/utils/makefile.python b/utils/makefile.python index 12e3e1236..345434384 100644 --- a/utils/makefile.python +++ b/utils/makefile.python @@ -75,7 +75,9 @@ msg-python-exe: https://www.python.org or install it from your package\n\ manager. On debian based OS these requirements are\n\ installed by::\n\n\ - sudo -H apt-get install $(PYTHON)\n" | $(FMT) + sudo -H add-apt-repository ppa:deadsnakes/ppa\n\ + sudo -H apt update\n\ + sudo -H apt-get install $(PYTHON) $(PYTHON)-venv\n" ifeq ($(shell which $(PYTHON) >/dev/null 2>&1; echo $$?), 1) python-exe: msg-python-exe @@ -95,7 +97,13 @@ quiet_cmd_pyinstall = INSTALL $2 # $2 path to folder with setup.py, this uses pip from pyenv (not OS!) quiet_cmd_pyenvinstall = PYENV install $2 - cmd_pyenvinstall = $(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) install -e $2$(PY_SETUP_EXTRAS) + cmd_pyenvinstall = \ + if ! cat $(PY_ENV)/requirements.sha256 2>/dev/null | sha256sum --check --status 2>/dev/null; then \ + $(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) install -e $2$(PY_SETUP_EXTRAS) ;\ + sha256sum requirements*.txt > $(PY_ENV)/requirements.sha256 ;\ + else \ + echo "PYENV $2 already installed"; \ + fi # Uninstall the package. Since pip does not uninstall the no longer needed # depencies (something like autoremove) the depencies remain. @@ -113,13 +121,15 @@ quiet_cmd_virtualenv = PYENV usage: $ source ./$@/bin/activate cmd_virtualenv = \ if [ ! -d "./$(PY_ENV)" ];then \ $(PYTHON) -m venv $(VTENV_OPTS) $2; \ + $(PY_ENV_BIN)/python -m pip install $(PIP_VERBOSE) -U pip wheel setuptools; \ + $(PY_ENV_BIN)/python -m pip install $(PIP_VERBOSE) -r requirements.txt; \ else \ echo "PYENV using virtualenv from $2"; \ fi # $2 path to lint quiet_cmd_pylint = LINT $@ - cmd_pylint = $(PY_ENV_BIN)/python -m pylint --rcfile $(PYLINT_RC) $2 + cmd_pylint = $(PY_ENV_BIN)/python -m pylint -j 0 --rcfile $(PYLINT_RC) $2 quiet_cmd_pytest = TEST $@ cmd_pytest = $(PY_ENV_BIN)/python -m tox -vv @@ -193,8 +203,6 @@ pyclean: pyenv: $(PY_ENV) $(PY_ENV): python-exe $(call cmd,virtualenv,$(PY_ENV)) - $(Q)$(PY_ENV_BIN)/python -m pip install $(PIP_VERBOSE) -U pip wheel pip setuptools - $(Q)$(PY_ENV_BIN)/python -m pip install $(PIP_VERBOSE) -r requirements.txt PHONY += pylint-exe pylint-exe: $(PY_ENV) @@ -220,16 +228,8 @@ PHONY += pydebug pydebug: $(PY_ENV) DEBUG=$(DEBUG) $(PY_ENV_BIN)/pytest $(DEBUG) -v $(TEST_FOLDER)/$(TEST) -# install / uninstall python objects into virtualenv (PYENV) -pyenv-install: $(PY_ENV) - @$(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) install -e . - @echo "ACTIVATE $(call normpath,$(PY_ENV_ACT)) " - -pyenv-uninstall: $(PY_ENV) - @$(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) uninstall --yes . - # runs python interpreter from ./local/py<N>/bin/python -pyenv-python: pyenv-install +pyenv-python: pyenvinstall $(PY_ENV_BIN)/python -i # With 'dependency_links=' setuptools supports dependencies on packages hosted @@ -243,8 +243,10 @@ pyenv-python: pyenv-install # [2] https://github.com/pypa/pip/pull/1519 # https://github.com/pypa/twine -PHONY += upload-pypi +PHONY += upload-pypi upload-pypi-test upload-pypi: pyclean pyenvinstall pybuild @$(PY_ENV_BIN)/twine upload $(PYDIST)/* +upload-pypi-test: pyclean pyenvinstall pybuild + @$(PY_ENV_BIN)/twine upload -r testpypi $(PYDIST)/* .PHONY: $(PHONY) diff --git a/utils/makefile.sphinx b/utils/makefile.sphinx index b4f298d50..e73686466 100644 --- a/utils/makefile.sphinx +++ b/utils/makefile.sphinx @@ -27,7 +27,7 @@ docs-help: @echo ' docs-clean - clean intermediate doc objects' @echo ' $(GH_PAGES) - create & upload github pages' @echo ' sphinx-pdf - run sphinx latex & pdf targets' - echo '' + @echo '' @echo ' books/{name}.html : build only the HTML of document {name}' @echo ' valid values for books/{name}.html are:' @echo ' $(BOOKS_HTML)' | $(FMT) @@ -99,7 +99,7 @@ sphinx-pdf: sphinx-latex @echo "SPHINX *.pdf --> file://$(abspath $(DOCS_DIST)/pdf)" PHONY += sphinx-latex -sphinx-latex: texlive sphinx-doc +sphinx-latex: pyenvinstall texlive $(SPHINXBUILD) $(SPHINX_VERBOSE) $(SPHINXOPTS)\ -b latex \ -c $(DOCS_FOLDER) \ @@ -124,7 +124,7 @@ $(BOOKS_DIST): mkdir -p $(BOOKS_DIST) PHONY += $(BOOKS_HTML) -$(BOOKS_HTML): sphinx-doc | $(BOOKS_DIST) +$(BOOKS_HTML): pyenvinstall | $(BOOKS_DIST) SPHINX_CONF=$(patsubst books/%.html,%,$@)/conf.py \ $(SPHINXBUILD) $(SPHINX_VERBOSE) $(SPHINXOPTS)\ -b html \ @@ -135,7 +135,7 @@ $(BOOKS_HTML): sphinx-doc | $(BOOKS_DIST) @echo "SPHINX $@ --> file://$(abspath $(BOOKS_DIST)/$(patsubst books/%.html,%,$@))" PHONY += $(BOOKS_HTML) -$(BOOKS_LIVE): sphinx-live | $(BOOKS_DIST) +$(BOOKS_LIVE): pyenvinstall | $(BOOKS_DIST) PATH="$(PY_ENV_BIN):$(PATH)" \ SPHINX_CONF=$(patsubst books/%.live,%,$@)/conf.py \ $(PY_ENV_BIN)/sphinx-autobuild --poll -B --host 0.0.0.0 --port 8080 $(SPHINX_VERBOSE) $(SPHINXOPTS)\ @@ -152,7 +152,7 @@ $(BOOKS_PDF): %.pdf : %.latex @echo "SPHINX $@ --> file://$(abspath $(BOOKS_DIST)/$(patsubst books/%.pdf,%,$@))/pdf" PHONY += $(BOOKS_LATEX) -$(BOOKS_LATEX): sphinx-doc | $(BOOKS_DIST) +$(BOOKS_LATEX): pyenvinstall | $(BOOKS_DIST) SPHINX_CONF=$(patsubst books/%.latex,%,$@)/conf.py \ $(SPHINXBUILD) $(SPHINX_VERBOSE) $(SPHINXOPTS)\ -b latex \ diff --git a/utils/searx.sh b/utils/searx.sh index 5a24a016e..b7d3b8e1c 100755 --- a/utils/searx.sh +++ b/utils/searx.sh @@ -36,7 +36,7 @@ GIT_BRANCH="${GIT_BRANCH:-master}" SEARX_PYENV="${SERVICE_HOME}/searx-pyenv" SEARX_SRC="${SERVICE_HOME}/searx-src" SEARX_SETTINGS_PATH="/etc/searx/settings.yml" -SEARX_SETTINGS_TEMPLATE="${REPO_ROOT}/utils/templates/etc/searx/use_default_settings.yml" +SEARX_SETTINGS_TEMPLATE="${SEARX_SETTINGS_TEMPLATE:-${REPO_ROOT}/utils/templates/etc/searx/use_default_settings.yml}" SEARX_UWSGI_APP="searx.ini" # shellcheck disable=SC2034 SEARX_UWSGI_SOCKET="/run/uwsgi/app/searx/socket" @@ -50,7 +50,7 @@ shellcheck" BUILD_PACKAGES_debian="\ firefox graphviz imagemagick texlive-xetex librsvg2-bin -texlive-latex-recommended texlive-extra-utils ttf-dejavu +texlive-latex-recommended texlive-extra-utils fonts-dejavu latexmk" # pacman packages @@ -157,7 +157,7 @@ install / remove :searx-src: clone $GIT_URL :pyenv: create/remove virtualenv (python) in $SEARX_PYENV :uwsgi: install searx uWSGI application - :settings: reinstall settings from ${REPO_ROOT}/searx/settings.yml + :settings: reinstall settings from ${SEARX_SETTINGS_TEMPLATE} :packages: install needed packages from OS package manager :buildhost: install packages from OS package manager needed by buildhosts update searx @@ -388,6 +388,14 @@ clone_searx() { err_msg "to clone searx sources, user $SERVICE_USER hast to be created first" return 42 fi + if [[ ! $(git show-ref "refs/heads/${GIT_BRANCH}") ]]; then + warn_msg "missing local branch ${GIT_BRANCH}" + info_msg "create local branch ${GIT_BRANCH} from start point: origin/${GIT_BRANCH}" + git branch "${GIT_BRANCH}" "origin/${GIT_BRANCH}" + fi + if [[ ! $(git branch --show-current) == "${GIT_BRANCH}" ]]; then + warn_msg "take into account, installing branch $GIT_BRANCH while current branch is $(git branch --show-current)" + fi export SERVICE_HOME git_clone "$REPO_ROOT" "$SEARX_SRC" \ "$GIT_BRANCH" "$SERVICE_USER" @@ -412,7 +420,7 @@ install_settings() { mkdir -p "$(dirname ${SEARX_SETTINGS_PATH})" if [[ ! -f ${SEARX_SETTINGS_PATH} ]]; then - info_msg "install settings ${REPO_ROOT}/searx/settings.yml" + info_msg "install settings ${SEARX_SETTINGS_TEMPLATE}" info_msg " --> ${SEARX_SETTINGS_PATH}" cp "${SEARX_SETTINGS_TEMPLATE}" "${SEARX_SETTINGS_PATH}" configure_searx diff --git a/utils/standalone_searx.py b/utils/standalone_searx.py index c52035fb9..89023f41b 100755 --- a/utils/standalone_searx.py +++ b/utils/standalone_searx.py @@ -15,7 +15,7 @@ Example to use this script: .. code:: bash - $ SEARX_DEBUG=1 python3 utils/standalone_searx.py rain + $ python3 utils/standalone_searx.py rain Example to run it from python: |