diff options
| -rwxr-xr-x | manage | 506 |
1 files changed, 506 insertions, 0 deletions
@@ -0,0 +1,506 @@ +#!/usr/bin/env bash +# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- +# SPDX-License-Identifier: AGPL-3.0-or-later +# shellcheck disable=SC2031 + +# shellcheck source=utils/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh" +# shellcheck source=utils/brand.env +source "${REPO_ROOT}/utils/brand.env" +source_dot_config + +# config + +PY_SETUP_EXTRAS='[test]' +NPM_PACKAGES="less@2.7 less-plugin-clean-css grunt-cli" +GECKODRIVER_VERSION="v0.28.0" +# SPHINXOPTS= + +# These py files are linted by test.pylint(), all other files are linted by +# test.pep8() +PYLINT_FILES=( + searx/preferences.py + searx/testing.py + searx/engines/gigablast.py + searx/engines/deviantart.py + searx/engines/digg.py + searx/engines/google.py + searx/engines/google_news.py + searx/engines/google_videos.py + searx/engines/google_images.py + searx/engines/mediathekviewweb.py +) + +PYLINT_SEARX_DISABLE_OPTION="\ +I,C,R,\ +W0105,W0212,W0511,W0603,W0613,W0621,W0702,W0703,W1401,\ +E1136" +PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES="supported_languages,language_aliases" +PYLINT_OPTIONS="-m pylint -j 0 --rcfile .pylintrc" + +help() { + cat <<EOF +buildenv + rebuild ./utils/brand.env +babel.compile + pybabel compile ./searx/translations +data.* + all : update searx/languages.py and ./data/* + languages : update searx/data/engines_languages.json & searx/languages.py + useragents: update searx/data/useragents.json with the most recent versions of Firefox. +docs.* + html : build HTML documentation + live : autobuild HTML documentation while editing + gh-pages : deploy on gh-pages branch + prebuild : build reST include files (./${DOCS_BUILD}/includes) + clean : clean documentation build +docker + build : build docker image + push : build and push docker image +gecko.driver + download & install geckodriver if not already installed (required for + robot_tests) +node.* + env : download & install npm dependencies locally + clean : drop npm installations +py.* + build : Build python packages at ./${PYDIST} + clean : delete virtualenv and intermediate py files +pyenv.* : + install : developer install of searx into virtualenv + uninstall : uninstall developer installation + cmd ... : run command ... in virtualenv + OK : test if virtualenv is OK +pypi.upload: + Upload python packages to PyPi (to test use pypi.upload.test) +test.* : + pylint : lint PYLINT_FILES, searx/engines, searx & tests + pep8 : pycodestyle (pep8) for all files except PYLINT_FILES + unit : run unit tests + coverage : run unit tests with coverage + robot : run robot test + clean : clean intermediate test stuff +themes.* : + all : build all themes + oscar : build oscar theme + simple : build simple theme + bootstrap : less compile bootstrap.min.css CSS +EOF +} + + +if [ "$VERBOSE" = "1" ]; then + SPHINX_VERBOSE="-v" + PYLINT_VERBOSE="-v" +fi + +# needed by sphinx-docs +export DOCS_BUILD + +buildenv() { + SEARX_DEBUG=1 pyenv.cmd python utils/build_env.py 2>&1 \ + | prefix_stdout "${_Blue}BUILDENV${_creset} " + return "${PIPESTATUS[0]}" +} + +babel.compile() { + build_msg BABEL compile + pyenv.cmd pybabel compile -d "${REPO_ROOT}/searx/translations" + dump_return $? +} + + +data.all() { + data.languages + data.useragents + build_msg DATA "update searx/data/ahmia_blacklist.txt" + pyenv.cmd python utils/fetch_ahmia_blacklist.py + build_msg DATA "update searx/data/wikidata_units.json" + pyenv.cmd python utils/fetch_wikidata_units.py + build_msg DATA "update searx/data/currencies.json" + pyenv.cmd python utils/fetch_currencies.py +} + + +data.languages() { + ( set -e + build_msg ENGINES "fetch languages .." + pyenv.cmd python utils/fetch_languages.py + build_msg ENGINES "update update searx/languages.py" + build_msg DATA "update searx/data/engines_languages.json" + ) + dump_return $? +} + +data.useragents() { + build_msg DATA "update searx/data/useragents.json" + pyenv.cmd python utils/fetch_firefox_version.py + dump_return $? +} + +docs.prebuild() { + build_msg DOCS "build ${DOCS_BUILD}/includes" + ( + set -e + [ "$VERBOSE" = "1" ] && set -x + mkdir -p "${DOCS_BUILD}/includes" + ./utils/searx.sh doc | cat > "${DOCS_BUILD}/includes/searx.rst" + ./utils/filtron.sh doc | cat > "${DOCS_BUILD}/includes/filtron.rst" + ./utils/morty.sh doc | cat > "${DOCS_BUILD}/includes/morty.rst" + ) + dump_return $? +} + +docker.push() { + docker.build push +} + +# shellcheck disable=SC2119 +docker.build() { + pyenv.install + + build_msg DOCKER build + # run installation in a subprocess and activate pyenv + + # See https://www.shellcheck.net/wiki/SC1001 and others .. + # shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001 + ( set -e + # shellcheck source=/dev/null + source "${PY_ENV_BIN}/activate" + + # Check if it is a git repository + if [ ! -d .git ]; then + die 1 "This is not Git repository" + fi + if [ ! -x "$(which git)" ]; then + die 1 "git is not installed" + fi + + if ! git remote get-url origin 2> /dev/null; then + die 1 "there is no remote origin" + fi + + # This is a git repository + + # "git describe" to get the Docker version (for example : v0.15.0-89-g0585788e) + # awk to remove the "v" and the "g" + SEARX_GIT_VERSION=$(git describe --match "v[0-9]*\.[0-9]*\.[0-9]*" HEAD 2>/dev/null | awk -F'-' '{OFS="-"; $1=substr($1, 2); if ($3) { $3=substr($3, 2); } print}') + + # add the suffix "-dirty" if the repository has uncommited change + # /!\ HACK for searx/searx: ignore utils/brand.env + git update-index -q --refresh + if [ ! -z "$(git diff-index --name-only HEAD -- | grep -v 'utils/brand.env')" ]; then + SEARX_GIT_VERSION="${SEARX_GIT_VERSION}-dirty" + fi + + # Get the last git commit id, will be added to the Searx version (see Dockerfile) + VERSION_GITCOMMIT=$(echo "$SEARX_GIT_VERSION" | cut -d- -f2-4) + build_msg DOCKER "Last commit : $VERSION_GITCOMMIT" + + # Check consistency between the git tag and the searx/version.py file + # /! HACK : parse Python file with bash /! + # otherwise it is not possible build the docker image without all Python + # dependencies ( version.py loads __init__.py ) + # SEARX_PYTHON_VERSION=$(python3 -c "import six; import searx.version; six.print_(searx.version.VERSION_STRING)") + SEARX_PYTHON_VERSION=$(cat searx/version.py | grep "\(VERSION_MAJOR\|VERSION_MINOR\|VERSION_BUILD\) =" | cut -d\= -f2 | sed -e 's/^[[:space:]]*//' | paste -sd "." -) + if [ "$(echo "$SEARX_GIT_VERSION" | cut -d- -f1)" != "$SEARX_PYTHON_VERSION" ]; then + err_msg "git tag: $SEARX_GIT_VERSION" + err_msg "searx/version.py: $SEARX_PYTHON_VERSION" + die 1 "Inconsistency between the last git tag and the searx/version.py file" + fi + + # define the docker image name + GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/\([^\/]*\).*/\1/') + SEARX_IMAGE_NAME="${SEARX_IMAGE_NAME:-${GITHUB_USER:-searx}/searx}" + + # build Docker image + build_msg DOCKER "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}" + sudo docker build \ + --build-arg GIT_URL="${GIT_URL}" \ + --build-arg SEARX_GIT_VERSION="${SEARX_GIT_VERSION}" \ + --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \ + --build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ + --build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \ + --build-arg LABEL_VCS_URL="${GIT_URL}" \ + --build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \ + --build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \ + -t "${SEARX_IMAGE_NAME}:latest" -t "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}" . + + if [ "$1" = "push" ]; then + sudo docker push "${SEARX_IMAGE_NAME}:latest" + sudo docker push "${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}" + fi + ) + dump_return $? +} + +# shellcheck disable=SC2119 +gecko.driver() { + pyenv.install + + build_msg INSTALL "gecko.driver" + # run installation in a subprocess and activate pyenv + ( set -e + # shellcheck source=/dev/null + source "${PY_ENV_BIN}/activate" + + # TODO : check the current geckodriver version + geckodriver -V > /dev/null 2>&1 || NOTFOUND=1 + set +e + if [ -z "$NOTFOUND" ]; then + build_msg INSTALL "geckodriver already installed" + return + fi + PLATFORM="$(python3 -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')" + case "$PLATFORM" in + "linux 32bit" | "linux2 32bit") ARCH="linux32";; + "linux 64bit" | "linux2 64bit") ARCH="linux64";; + "windows 32 bit") ARCH="win32";; + "windows 64 bit") ARCH="win64";; + "mac 64bit") ARCH="macos";; + esac + GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz"; + + build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL" + + FILE="$(mktemp)" + wget -qO "$FILE" -- "$GECKODRIVER_URL" && tar xz -C "${PY_ENV_BIN}" -f "$FILE" geckodriver + rm -- "$FILE" + chmod 755 -- "${PY_ENV_BIN}/geckodriver" + ) + dump_return $? +} + +node.env() { + local err=0 + pushd "${REPO_ROOT}" &> /dev/null + # shellcheck disable=SC2230 + which npm &> /dev/null || die 1 'node.env - npm is not found!' + + ( set -e + # shellcheck disable=SC2030 + PATH="$(npm bin):$PATH" + export PATH + + build_msg INSTALL "npm install $NPM_PACKAGES" + # shellcheck disable=SC2086 + npm install $NPM_PACKAGES + + cd "${REPO_ROOT}/searx/static/themes/oscar" + build_msg INSTALL "($(pwd)) npm install" + npm install + + build_msg INSTALL "($(pwd)) npm install" + cd "${REPO_ROOT}/searx/static/themes/simple" + npm install + ) + err=$? + popd &> /dev/null + dump_return "$err" +} + +node.clean() { + + build_msg CLEAN "locally installed npm dependencies" + rm -rf \ + ./node_modules \ + ./package-lock.json \ + ./searx/static/themes/oscar/package-lock.json \ + ./searx/static/themes/oscar/node_modules \ + ./searx/static/themes/simple/package-lock.json \ + ./searx/static/themes/simple/node_modules + dump_return $? +} + +py.build() { + build_msg BUILD "[pylint] python package ${PYDIST}" + pyenv.cmd python setup.py \ + sdist -d "${PYDIST}" \ + bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}" +} + +py.clean() { + build_msg CLEAN pyenv + ( set -e + pyenv.drop + [ "$VERBOSE" = "1" ] && set -x + rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info + find . -name '*.pyc' -exec rm -f {} + + find . -name '*.pyo' -exec rm -f {} + + find . -name __pycache__ -exec rm -rf {} + + ) +} + +pyenv.check() { + cat <<EOF +import yaml +print('import yaml --> OK') +EOF +} + +pyenv.install() { + + if ! pyenv.OK; then + py.clean > /dev/null + fi + if pyenv.install.OK > /dev/null; then + return 0 + fi + ( set -e + py.clean > /dev/null + pyenv + build_msg PYENV "[install] pip install -e 'searx${PY_SETUP_EXTRAS}'" + "${PY_ENV_BIN}/python" -m pip install -e ".${PY_SETUP_EXTRAS}" + buildenv + ) || die 42 "error while build & install pyenv (${PY_ENV_BIN})" +} + +pyenv.uninstall() { + build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}" + pyenv.cmd python setup.py develop --uninstall 2>&1 \ + | prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] " + +} + +pypi.upload() { + py.clean + py.build + # https://github.com/pypa/twine + pyenv.cmd twine upload "${PYDIST}"/* +} + +pypi.upload.test() { + py.clean + py.build + pyenv.cmd twine upload -r testpypi "${PYDIST}"/* +} + +test.pylint() { + # shellcheck disable=SC2086 + ( set -e + build_msg TEST "[pylint] \$PYLINT_FILES" + pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \ + "${PYLINT_FILES[@]}" + + build_msg TEST "[pylint] searx/engines" + pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \ + --disable="${PYLINT_SEARX_DISABLE_OPTION}" \ + --additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \ + searx/engines + + build_msg TEST "[pylint] searx tests" + pyenv.cmd python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \ + --disable="${PYLINT_SEARX_DISABLE_OPTION}" \ + --ignore=searx/engines \ + searx tests + ) + dump_return $? +} + +test.pep8() { + build_msg TEST 'pycodestyle (formerly pep8)' + local _exclude="" + printf -v _exclude '%s, ' "${PYLINT_FILES[@]}" + pyenv.cmd pycodestyle \ + --exclude="searx/static, searx/languages.py, $_exclude " \ + --max-line-length=120 \ + --ignore "E117,E252,E402,E722,E741,W503,W504,W605" \ + searx tests + dump_return $? +} + +test.unit() { + build_msg TEST 'tests/unit' + pyenv.cmd python -m nose2 -s tests/unit + dump_return $? +} + +test.coverage() { + build_msg TEST 'unit test coverage' + ( set -e + pyenv.cmd python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit + pyenv.cmd coverage report + pyenv.cmd coverage html + ) + dump_return $? +} + +test.robot() { + build_msg TEST 'robot' + gecko.driver + PYTHONPATH=. pyenv.cmd python searx/testing.py robot + dump_return $? +} + +test.clean() { + build_msg CLEAN "test stuff" + rm -rf geckodriver.log .coverage coverage/ + dump_return $? +} + +themes.all() { + ( set -e + node.env + themes.oscar + themes.simple + themes.bootstrap + ) + dump_return $? +} + +themes.oscar() { + local gruntfile=searx/static/themes/oscar/gruntfile.js + build_msg GRUNT "${gruntfile}" + PATH="$(npm bin):$PATH" grunt --gruntfile "${gruntfile}" + dump_return $? +} + +themes.simple() { + local gruntfile=searx/static/themes/simple/gruntfile.js + build_msg GRUNT "${gruntfile}" + PATH="$(npm bin):$PATH" grunt --gruntfile "${gruntfile}" + dump_return $? +} + +themes.bootstrap() { + build_msg LESSC "searx/static/css/bootstrap.min.css" + PATH="$(npm bin):$PATH" lessc \ + --clean-css="--s1 --advanced --compatibility=ie9" \ + "searx/static/less/bootstrap/bootstrap.less" \ + "searx/static/css/bootstrap.min.css" + dump_return $? +} + +# shellcheck disable=SC2119 +main() { + + local _type + local cmd="$1"; shift + + if [ "$cmd" == "" ]; then + help + err_msg "missing command" + return 42 + fi + + case "$cmd" in + --getenv) var="$1"; echo "${!var}";; + --help) help;; + --*) + help + err_msg "unknown option $cmd" + return 42 + ;; + *) + _type="$(type -t "$cmd")" + if [ "$_type" != 'function' ]; then + err_msg "unknown command $1 / use --help" + return 42 + else + "$cmd" "$@" + fi + ;; + esac +} + +main "$@" |