Merge branch 'master' into fix/manage.sh

5 files changed, 258 insertions, 61 deletions

diff --git a/utils/brand.env b/utils/brand.env
new file mode 100644
index 000000000..7fe1a3911
--- /dev/null
+++ b/utils/brand.env
@@ -0,0 +1,5 @@
+export GIT_URL='https://github.com/asciimoo/searx'
+export ISSUE_URL='https://github.com/asciimoo/searx/issues'
+export SEARX_URL='https://searx.me'
+export DOCS_URL='https://asciimoo.github.io/searx'
+export PUBLIC_INSTANCES='https://searx.space'
diff --git a/utils/fetch_firefox_version.py b/utils/fetch_firefox_version.py
index ed179585b..722c48229 100755
--- a/utils/fetch_firefox_version.py
+++ b/utils/fetch_firefox_version.py
@@ -24,7 +24,7 @@ NORMAL_REGEX = re.compile('^[0-9]+\.[0-9](\.[0-9])?$')
 # 
 useragents = {
     "versions": (),
-    "os": ('Windows NT 10; WOW64',
+    "os": ('Windows NT 10.0; WOW64',
            'X11; Linux x86_64'),
     "ua": "Mozilla/5.0 ({os}; rv:{version}) Gecko/20100101 Firefox/{version}"
 }
diff --git a/utils/fetch_languages.py b/utils/fetch_languages.py
index a6af07303..ddebb4111 100644
--- a/utils/fetch_languages.py
+++ b/utils/fetch_languages.py
@@ -5,7 +5,7 @@
 # Output files (engines_languages.json and languages.py)
 # are written in current directory to avoid overwriting in case something goes wrong.
 
-from json import dump
+import json
 import io
 from sys import path
 from babel import Locale, UnknownLocaleError
@@ -22,19 +22,22 @@ languages_file = 'languages.py'
 
 # Fetchs supported languages for each engine and writes json file with those.
 def fetch_supported_languages():
+
     engines_languages = {}
-    for engine_name in engines:
+    names = list(engines)
+    names.sort()
+
+    for engine_name in names:
+        print("fetching languages of engine %s" % engine_name)
+
         if hasattr(engines[engine_name], 'fetch_supported_languages'):
-            try:
-                engines_languages[engine_name] = engines[engine_name].fetch_supported_languages()
-                if type(engines_languages[engine_name]) == list:
-                    engines_languages[engine_name] = sorted(engines_languages[engine_name])
-            except Exception as e:
-                print(e)
+            engines_languages[engine_name] = engines[engine_name].fetch_supported_languages()
+            if type(engines_languages[engine_name]) == list:
+                engines_languages[engine_name] = sorted(engines_languages[engine_name])
 
     # write json file
-    with io.open(engines_languages_file, "w", encoding="utf-8") as f:
-        dump(engines_languages, f, ensure_ascii=False, indent=4, separators=(',', ': '))
+    with open(engines_languages_file, 'w', encoding='utf-8') as f:
+        json.dump(engines_languages, f, indent=2, sort_keys=True)
 
     return engines_languages
 
diff --git a/utils/makefile.python b/utils/makefile.python
index 228eb3f80..590bbdb46 100644
--- a/utils/makefile.python
+++ b/utils/makefile.python
@@ -5,6 +5,7 @@ PYOBJECTS ?=
 
 SITE_PYTHON ?=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))site-python
 export PYTHONPATH := $(SITE_PYTHON):$$PYTHONPATH
+export PY_ENV PYDIST PYBUILD
 
 # folder where the python distribution takes place
 PYDIST   ?= ./py_dist
@@ -12,6 +13,9 @@ PYDIST   ?= ./py_dist
 PYBUILD  ?= ./py_build
 # python version to use
 PY       ?=3
+# $(PYTHON) points to the python interpreter from the OS!  The python from the
+# OS is needed e.g. to create a virtualenv.  For tasks inside the virtualenv the
+# interpeter from '$(PY_ENV_BIN)/python' is used.
 PYTHON   ?= python$(PY)
 PIP      ?= pip$(PY)
 PIP_INST ?= --user
@@ -59,7 +63,7 @@ python-help::
 	@echo  '    pylint    - run pylint *linting*'
 	@echo  '    pytest    - run *tox* test on python objects'
 	@echo  '    pydebug   - run tests within a PDB debug session'
-	@echo  '    pybuild   - build python packages'
+	@echo  '    pybuild   - build python packages ($(PYDIST) $(PYBUILD))'
 	@echo  '    pyclean   - clean intermediate python objects'
 	@echo  '  targets using system users environment:'
 	@echo  '    py[un]install - [un]install python objects in editable mode'
@@ -94,38 +98,6 @@ python-exe:
 	@:
 endif
 
-msg-pip-exe:
-	@echo "\n  $(PIP) is required\n\n\
-  Make sure you have updated pip installed, grab it from\n\
-  https://pip.pypa.io or install it from your package\n\
-  manager. On debian based OS these requirements are\n\
-  installed by::\n\n\
-    sudo -H apt-get install python$(PY)-pip\n" | $(FMT)
-
-ifeq ($(shell which $(PIP) >/dev/null 2>&1; echo $$?), 1)
-pip-exe: msg-pip-exe
-	$(error The '$(PIP)' command was not found)
-else
-pip-exe:
-	@:
-endif
-
-PHONY += msg-virtualenv-exe virtualenv-exe
-msg-virtualenv-exe:
-	@echo "\n  virtualenv is required\n\n\
-  Make sure you have an updated virtualenv installed, grab it from\n\
-  https://virtualenv.pypa.io/en/stable/installation/ or install it\n\
-  via pip by::\n\n\
-    pip install --user https://github.com/pypa/virtualenv/tarball/master\n" | $(FMT)
-
-ifeq ($(shell which virtualenv >/dev/null 2>&1; echo $$?), 1)
-virtualenv-exe: msg-virtualenv-exe
-	$(error The 'virtualenv' command was not found)
-else
-virtualenv-exe:
-	@:
-endif
-
 # ------------------------------------------------------------------------------
 # commands
 # ------------------------------------------------------------------------------
@@ -136,9 +108,9 @@ quiet_cmd_pyinstall = INSTALL   $2
 
 # $2 path to folder with setup.py, this uses pip from pyenv (not OS!)
 quiet_cmd_pyenvinstall = PYENV     install $2
-      cmd_pyenvinstall = $(PY_ENV_BIN)/pip $(PIP_VERBOSE) install -e $2$(PY_SETUP_EXTRAS)
+      cmd_pyenvinstall = $(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) install -e $2$(PY_SETUP_EXTRAS)
 
-# Uninstall the package. Since pip does not uninstall the no longer needed
+# Uninstall the package.  Since pip does not uninstall the no longer needed
 # depencies (something like autoremove) the depencies remain.
 
 # $2 package name to uninstall, this uses pip from the OS.
@@ -147,7 +119,7 @@ quiet_cmd_pyuninstall = UNINSTALL $2
 
 # $2 path to folder with setup.py, this uses pip from pyenv (not OS!)
 quiet_cmd_pyenvuninstall = PYENV     uninstall   $2
-      cmd_pyenvuninstall = $(PY_ENV_BIN)/pip $(PIP_VERBOSE) uninstall --yes $2
+      cmd_pyenvuninstall = $(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) uninstall --yes $2
 
 # $2 path to folder where virtualenv take place
 quiet_cmd_virtualenv  = PYENV     usage: $ source ./$@/bin/activate
@@ -155,15 +127,15 @@ quiet_cmd_virtualenv  = PYENV     usage: $ source ./$@/bin/activate
 	if [ ! -d "./$(PY_ENV)" ];then                                  \
 		$(VIRTUALENV) $(VIRTUALENV_VERBOSE) $(VTENV_OPTS) $2;   \
 	else                                                            \
-		echo "  PYENV     using virtualenv from $2";            \
+		echo "PYENV     using virtualenv from $2";              \
         fi
 
 # $2 path to lint
 quiet_cmd_pylint      = LINT      $@
-      cmd_pylint      = $(PY_ENV_BIN)/pylint --rcfile $(PYLINT_RC) $2
+      cmd_pylint      = $(PY_ENV_BIN)/python -m pylint --rcfile $(PYLINT_RC) $2
 
 quiet_cmd_pytest      = TEST      $@
-      cmd_pytest      = $(PY_ENV_BIN)/tox -vv
+      cmd_pytest      = $(PY_ENV_BIN)/python -m tox -vv
 
 # setuptools, pip, easy_install its a mess full of cracks, a documentation hell
 # and broken by design ... all sucks, I really, really hate all this ... aaargh!
@@ -192,14 +164,14 @@ quiet_cmd_pytest      = TEST      $@
 # .. _installing: https://packaging.python.org/tutorials/installing-packages/
 #
 quiet_cmd_pybuild     = BUILD     $@
-      cmd_pybuild     = $(PY_ENV_BIN)/$(PYTHON) setup.py \
+      cmd_pybuild     = $(PY_ENV_BIN)/python setup.py \
 			sdist -d $(PYDIST)  \
 			bdist_wheel --bdist-dir $(PYBUILD) -d $(PYDIST)
 
 quiet_cmd_pyclean     = CLEAN     $@
 # remove 'build' folder since bdist_wheel does not care the --bdist-dir
       cmd_pyclean     = \
-	rm -rf $(PYDIST) $(PYBUILD) ./local ./.tox *.egg-info     ;\
+	rm -rf $(PYDIST) $(PYBUILD) $(PY_ENV) ./.tox *.egg-info     ;\
 	find . -name '*.pyc' -exec rm -f {} +      ;\
 	find . -name '*.pyo' -exec rm -f {} +      ;\
 	find . -name __pycache__ -exec rm -rf {} +
@@ -230,15 +202,16 @@ PHONY += pyclean
 pyclean:
 	$(call cmd,pyclean)
 
-# to build *local* environment, python and virtualenv from the OS is needed!
+# to build *local* environment, python from the OS is needed!
 pyenv: $(PY_ENV)
-$(PY_ENV): virtualenv-exe python-exe
+$(PY_ENV): python-exe
 	$(call cmd,virtualenv,$(PY_ENV))
-	@$(PY_ENV_BIN)/pip install $(PIP_VERBOSE) -r requirements.txt
+	$(Q)$(PY_ENV_BIN)/python -m pip install $(PIP_VERBOSE) -U pip wheel pip setuptools
+	$(Q)$(PY_ENV_BIN)/python -m pip install $(PIP_VERBOSE) -r requirements.txt
 
 PHONY += pylint-exe
 pylint-exe: $(PY_ENV)
-	@$(PY_ENV_BIN)/pip $(PIP_VERBOSE) install pylint
+	@$(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) install pylint
 
 PHONY += pylint
 pylint: pylint-exe
@@ -262,15 +235,15 @@ pydebug: $(PY_ENV)
 
 # install / uninstall python objects into virtualenv (PYENV)
 pyenv-install: $(PY_ENV)
-	@$(PY_ENV_BIN)/pip $(PIP_VERBOSE) install -e .
-	@echo "  ACTIVATE  $(call normpath,$(PY_ENV_ACT)) "
+	@$(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) install -e .
+	@echo "ACTIVATE  $(call normpath,$(PY_ENV_ACT)) "
 
 pyenv-uninstall: $(PY_ENV)
-	@$(PY_ENV_BIN)/pip $(PIP_VERBOSE) uninstall --yes .
+	@$(PY_ENV_BIN)/python -m pip $(PIP_VERBOSE) uninstall --yes .
 
 # runs python interpreter from ./local/py<N>/bin/python
 pyenv-python: pyenv-install
-	cd ./local; ../$(PY_ENV_BIN)/python -i
+	$(PY_ENV_BIN)/python -i
 
 # With 'dependency_links=' setuptools supports dependencies on packages hosted
 # on other reposetories then PyPi, see "Packages Not On PyPI" [1].  The big
@@ -284,7 +257,7 @@ pyenv-python: pyenv-install
 
 # https://github.com/pypa/twine
 PHONY += upload-pypi
-upload-pypi: pyclean pybuild
+upload-pypi: pyclean pyenvinstall pybuild
 	@$(PY_ENV_BIN)/twine upload $(PYDIST)/*
 
 .PHONY: $(PHONY)
diff --git a/utils/makefile.sphinx b/utils/makefile.sphinx
new file mode 100644
index 000000000..2c1922fc9
--- /dev/null
+++ b/utils/makefile.sphinx
@@ -0,0 +1,216 @@
+# -*- coding: utf-8; mode: makefile-gmake -*-
+
+# You can set these variables from the command line.
+SPHINXOPTS  ?=
+SPHINXBUILD ?= $(PY_ENV_BIN)/sphinx-build
+SPHINX_CONF ?= conf.py
+
+DOCS_FOLDER ?= docs
+DOCS_BUILD  ?= build/docs
+DOCS_DIST   ?= dist/docs
+GH_PAGES    ?= gh-pages
+
+BOOKS_FOLDER ?= docs
+BOOKS_DIST   ?= dist/books
+
+ifeq ($(KBUILD_VERBOSE),1)
+  SPHINX_VERBOSE = "-v"
+else
+  SPHINX_VERBOSE =
+endif
+
+## SPHINXVERS variable
+## ===================
+##
+## .. _requirement-specifiers: https://pip.pypa.io/en/stable/reference/pip_install/#requirement-specifiers
+##
+## Sphinx version to use, when building documentation.  Set this when calling
+## build target.  The default value is empty (install latest), to select a
+## specific version use a requirement-specifiers_.  E.g. to build your target
+## 'doc' with a select sphinx-doc_ version 1.7.9::
+##
+##     make SPHINXVERS='==1.7.9' docs
+##
+## To build with latest 1.7::
+##
+##     make SPHINXVERS='>=1.7,<1.8' docs
+##
+SPHINXVERS  ?=
+
+docs-help:
+	@echo  'makefile.sphinx:'
+	@echo  '  docs-clean	- clean intermediate doc objects'
+	@echo  '  $(GH_PAGES)	- create & upload github pages'
+	@echo  '  sphinx-pdf    - run sphinx latex & pdf targets'
+	echo   ''
+	@echo  '  books/{name}.html : build only the HTML of document {name}'
+	@echo  '  valid values for books/{name}.html are:'
+	@echo  '    $(BOOKS_HTML)' | $(FMT)
+	@echo  '  books/{name}.pdf : build only the PDF of document {name}'
+	@echo  '  valid values for books/{name}.pdf are:'
+	@echo  '    $(BOOKS_PDF) ' | $(FMT)
+
+# ------------------------------------------------------------------------------
+# requirements
+# ------------------------------------------------------------------------------
+
+sphinx-doc: $(PY_ENV)
+	@echo "PYENV     installing Sphinx$(SPHINXVERS)"
+	$(Q)$(PY_ENV_BIN)/pip install $(PIP_VERBOSE) 'Sphinx$(SPHINXVERS)'
+
+sphinx-live: $(PY_ENV)
+	@echo "PYENV     installing Sphinx$(SPHINXVERS)"
+	$(Q)$(PY_ENV_BIN)/pip install $(PIP_VERBOSE) 'Sphinx$(SPHINXVERS)' sphinx-autobuild
+
+
+PHONY += msg-texlive texlive
+
+ifeq ($(shell which xelatex >/dev/null 2>&1; echo $$?), 1)
+texlive: msg-TeXLive
+	$(error The 'xelatex' command was not found)
+else
+texlive:
+	@:
+endif
+
+msg-texlive:
+	$(Q)echo "\n\
+The TeX/PDF output and the *math* extension require TexLive and latexmk:\n\n\
+  Make sure you have a updated TeXLive with XeTeX engine installed, grab it\n\
+  it from https://www.tug.org/texlive or install it from your package manager.\n\n\
+  Install latexmk from your package manager or visit https://ctan.org/pkg/latexmk\n\n\
+  Sphinx-doc produce (Xe)LaTeX files which might use additional TeX-packages\n\
+  and fonts. To process these LaTeX files, a TexLive installation with the\n\
+  additional packages is required. On debian based OS these requirements\n\
+  are installed by::\n\n\
+    sudo -H apt-get install\n\
+         latexmk\n\
+         texlive-base texlive-xetex texlive-latex-recommended\n\
+         texlive-extra-utils dvipng ttf-dejavu\n"
+
+# ------------------------------------------------------------------------------
+# commands
+# ------------------------------------------------------------------------------
+
+# $2 sphinx builder e.g. "html"
+# $3 path where configuration file (conf.py) is located
+# $4 sourcedir
+# $5 dest subfolder e.g. "man" for man pages at $(DOCS_DIST)/man
+
+quiet_cmd_sphinx = SPHINX    $@ --> file://$(abspath $(DOCS_DIST)/$5)
+      cmd_sphinx = SPHINX_CONF=$(abspath $4/$(SPHINX_CONF))\
+	$(SPHINXBUILD) $(SPHINX_VERBOSE) $(SPHINXOPTS)\
+	-b $2 -c $3 -d $(DOCS_BUILD)/.doctrees $4 $(DOCS_DIST)/$5
+
+quiet_cmd_sphinx_autobuild = SPHINX    $@ --> file://$(abspath $(DOCS_DIST)/$5)
+      cmd_sphinx_autobuild = PATH="$(PY_ENV_BIN):$(PATH)" $(PY_ENV_BIN)/sphinx-autobuild  $(SPHINX_VERBOSE) --poll -B --host 0.0.0.0 --port 8080 $(SPHINXOPTS)\
+	-b $2 -c $3 -d $(DOCS_BUILD)/.doctrees $4 $(DOCS_DIST)/$5
+
+quiet_cmd_sphinx_clean = CLEAN     $@
+      cmd_sphinx_clean = rm -rf $(DOCS_BUILD) $(DOCS_DIST) $(GH_PAGES)/* $(GH_PAGES)/.buildinfo
+
+# ------------------------------------------------------------------------------
+# targets
+# ------------------------------------------------------------------------------
+
+# build PDF of whole documentation in: $(DOCS_DIST)/pdf 
+
+PHONY += sphinx-pdf
+sphinx-pdf: sphinx-latex
+	$(Q)cd $(DOCS_BUILD)/latex/; make all-pdf
+	$(Q)mkdir -p $(DOCS_DIST)/pdf
+	$(Q)cp $(DOCS_BUILD)/latex/*.pdf $(DOCS_DIST)/pdf
+	@echo "SPHINX    *.pdf --> file://$(abspath $(DOCS_DIST)/pdf)"
+
+PHONY += sphinx-latex
+sphinx-latex: texlive sphinx-doc
+	$(SPHINXBUILD) $(SPHINX_VERBOSE) $(SPHINXOPTS)\
+	  -b latex \
+	  -c $(DOCS_FOLDER) \
+	  -d $(DOCS_BUILD)/.doctrees \
+	  $(DOCS_FOLDER) \
+	  $(DOCS_BUILD)/latex
+
+# Sphinx projects, we call them *books* (what is more common).  Books are
+# folders under $(BOOKS_FOLDER) containing a conf.py file. The HTML output goes
+# to folder $(BOOKS_DIST)/<name> while PDF is placed (BOOKS_DIST)/<name>/pdf
+
+BOOKS=$(patsubst $(BOOKS_FOLDER)/%/conf.py,books/%,$(wildcard $(BOOKS_FOLDER)/*/conf.py))
+
+# fine grained targets
+BOOKS_HTML  = $(patsubst %,%.html,$(BOOKS))
+BOOKS_CLEAN = $(patsubst %,%.clean,$(BOOKS))
+BOOKS_LATEX = $(patsubst %,%.latex,$(BOOKS))
+BOOKS_PDF   = $(patsubst %,%.pdf,$(BOOKS))
+BOOKS_LIVE  = $(patsubst %,%.live,$(BOOKS))
+
+$(BOOKS_DIST):
+	mkdir -p $(BOOKS_DIST)
+
+PHONY += $(BOOKS_HTML)
+$(BOOKS_HTML): sphinx-doc | $(BOOKS_DIST)
+	SPHINX_CONF=$(patsubst books/%.html,%,$@)/conf.py \
+	$(SPHINXBUILD) $(SPHINX_VERBOSE) $(SPHINXOPTS)\
+	  -b html \
+	  -c $(DOCS_FOLDER) \
+	  -d $(DOCS_BUILD)/books/$(patsubst books/%.html,%,$@)/.doctrees \
+	  $(patsubst books/%.html,%,$@) \
+	  $(BOOKS_DIST)/$(patsubst books/%.html,%,$@)
+	@echo "SPHINX    $@ --> file://$(abspath $(BOOKS_DIST)/$(patsubst books/%.html,%,$@))"
+
+PHONY += $(BOOKS_HTML)
+$(BOOKS_LIVE): sphinx-live | $(BOOKS_DIST)
+	PATH="$(PY_ENV_BIN):$(PATH)" \
+	SPHINX_CONF=$(patsubst books/%.live,%,$@)/conf.py \
+	$(PY_ENV_BIN)/sphinx-autobuild --poll -B --host 0.0.0.0 --port 8080 $(SPHINX_VERBOSE) $(SPHINXOPTS)\
+	  -b html \
+	  -c $(DOCS_FOLDER) \
+	  -d $(DOCS_BUILD)/books/$(patsubst books/%.live,%,$@)/.doctrees \
+	  $(patsubst books/%.live,%,$@) \
+	  $(BOOKS_DIST)/$(patsubst books/%.live,%,$@)
+
+$(BOOKS_PDF): %.pdf : %.latex
+	$(Q)cd $(DOCS_BUILD)/latex/$(patsubst books/%.pdf,%,$@); make all-pdf
+	$(Q)mkdir -p $(BOOKS_DIST)/$(patsubst books/%.pdf,%,$@)/pdf
+	$(Q)cp -v $(DOCS_BUILD)/latex/$(patsubst books/%.pdf,%,$@)/*.pdf $(BOOKS_DIST)/$(patsubst books/%.pdf,%,$@)/pdf
+	@echo "SPHINX    $@ --> file://$(abspath $(BOOKS_DIST)/$(patsubst books/%.pdf,%,$@))/pdf"
+
+PHONY += $(BOOKS_LATEX)
+$(BOOKS_LATEX): sphinx-doc | $(BOOKS_DIST)
+	SPHINX_CONF=$(patsubst books/%.latex,%,$@)/conf.py \
+	$(SPHINXBUILD) $(SPHINX_VERBOSE) $(SPHINXOPTS)\
+	  -b latex \
+	  -c $(DOCS_FOLDER) \
+	  -d $(DOCS_BUILD)/books/$(patsubst books/%.latex,%,$@)/.doctrees \
+	  $(patsubst books/%.latex,%,$@) \
+	  $(DOCS_BUILD)/latex/$(patsubst books/%.latex,%,$@)
+	@echo "SPHINX    $@ --> file://$(abspath $(DOCS_BUILD)/latex/$(patsubst books/%.latex,%,$@))"
+
+$(BOOKS_CLEAN):
+	$(Q)rm -rf $(BOOKS_DIST)/$(patsubst books/%.clean,%,$@) \
+	       $(DOCS_BUILD)/books/$(patsubst books/%.clean,%,$@) \
+	       $(DOCS_BUILD)/latex/$(patsubst books/%.clean,%,$@)
+
+# github pages
+
+PHONY += $(GH_PAGES)
+$(GH_PAGES)::
+	$(MAKE) docs
+	[ -d "gh-pages/.git" ] || git clone $(GIT_URL) gh-pages
+	-cd $(GH_PAGES); git checkout gh-pages >/dev/null
+	-cd $(GH_PAGES); git pull
+	-cd $(GH_PAGES); ls -A | grep -v '.git$$' | xargs rm -rf
+	cp -r $(DOCS_DIST)/* $(GH_PAGES)/
+	touch $(GH_PAGES)/.nojekyll
+	echo "<html><head><META http-equiv='refresh' content='0;URL=index.html'></head></html>" > $(GH_PAGES)/404.html
+	cd $(GH_PAGES);\
+		git add --all . ;\
+		git commit -m "gh-pages: updated" ;\
+		git push origin gh-pages
+
+
+PHONY += docs-clean
+docs-clean: $(BOOKS_CLEAN)
+	$(call cmd,sphinx_clean)
+
+.PHONY: $(PHONY)