Merge branch 'maint-0.4.2' into release-0.4.2

3 files changed, 533 insertions, 15 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e8d6c5f418..fc160ffaea 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,17 +1,111 @@
-image: debian:stable
+# This file controls how gitlab validates Tor commits and merge requests.
+#
+# It is primarily based on a set of scripts and configurations by
+# Hans-Christoph Steiner.  It only copies parts of those scripts and
+# configurations for now.  If you want a new piece of functionality
+# (more debians, more fedoras, android support) then you shouldn't
+# start from scratch: have a look at the original ticket, at
+# https://gitlab.torproject.org/tpo/core/tor/-/issues/32193 !
+#
+# The file to copy from is
+# https://gitlab.torproject.org/tpo/core/tor/-/merge_requests/96/diffs#diff-content-587d266bb27a4dc3022bbed44dfa19849df3044c
+#
+# Having said that, if there is anything really stupid here, don't
+# blame it on Hans-Christoph! Tor probably added it on their own.
+#
+# Copyright 2020, The Tor Project, Inc.
+# See LICENSE for licence information.
 
-before_script:
-    - apt-get update -qq
-    - apt-get upgrade -qy
+# These variables are set everywhere, unconditionally.
+variables:
+  TERM: "ansi"
+  DEBUG_CI: "yes"
 
-build:
+# This template is for exporting ephemeral things from the scripts.  By
+# convention we expect our scripts to copy stuff into artifacts/, rather than
+# having a big list of files that be treated as artifacts.
+.artifacts-template: &artifacts-template
+  artifacts:
+    name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}"
+    expire_in: 1 week
+    when: always
+    paths:
+      - artifacts/
+
+.apt-template: &apt-template |
+      export LC_ALL=C.UTF-8
+      echo Etc/UTC > /etc/timezone
+      mkdir -p apt-cache
+      export APT_CACHE_DIR="$(pwd)/apt-cache"
+      echo 'quiet "1";' \
+           'APT::Install-Recommends "0";' \
+           'APT::Install-Suggests "0";' \
+           'APT::Acquire::Retries "20";' \
+           'APT::Get::Assume-Yes "true";' \
+           'Dpkg::Use-Pty "0";' \
+           "Dir::Cache::Archives \"${APT_CACHE_DIR}\"; " \
+        >> /etc/apt/apt.conf.d/99gitlab
+      apt-get update -qq
+      apt-get upgrade -qy
+
+.debian-template: &debian-template
+  <<: *artifacts-template
+  variables:
+    DEBIAN_FRONTEND: "noninteractive"
+  cache:
+    key: apt
+    paths:
+      - apt-cache
+  before_script:
+    - *apt-template
+    - apt-get install
+        automake
+        build-essential
+        git
+        libevent-dev
+        liblzma-dev
+        libscrypt-dev
+        libseccomp-dev
+        libssl-dev
+        pkg-config
+        python3
+        zlib1g-dev
+    - if [ "$ASCIIDOC" = yes ]; then apt-get install asciidoc xmlto; fi
+    - if [ "$DOXYGEN" = yes ]; then apt-get install doxygen; fi
+
+debian-minimal:
+  image: debian:stable
+  <<: *debian-template
   script:
-    - apt-get install -qy --fix-missing automake build-essential
-      libevent-dev libssl-dev zlib1g-dev
-      libseccomp-dev liblzma-dev libscrypt-dev
-    - ./autogen.sh
-    - ./configure --disable-asciidoc --enable-fatal-warnings
-      --disable-silent-rules
-    - make check || (e=$?; cat test-suite.log; exit $e)
-    - make install
+    - ./scripts/ci/ci-driver.sh
+
+###############################################
+# Temporarily diabled. This one just takes too long to finish right now!
+# Maybe we need to divide the call to ./src/test/test into a few segments,
+# that all end in similar amount of time?
+#debian-hardened:
+#  image: debian:testing
+#  <<: *debian-template
+#  variables:
+#    HARDENING: "yes"
+#  script:
+#    - ./scripts/ci/ci-driver.sh
 
+debian-distcheck:
+  image: debian:stable
+  <<: *debian-template
+  variables:
+    DISTCHECK: "yes"
+    CHECK: "no"
+  script:
+    - ./scripts/ci/ci-driver.sh
+
+debian-docs:
+  image: debian:stable
+  <<: *debian-template
+  variables:
+    DOXYGEN: "no"
+    ASCIIDOC: "no"
+    CHECK: "no"
+  script:
+    - ./scripts/ci/ci-driver.sh
diff --git a/scripts/ci/ci-driver.sh b/scripts/ci/ci-driver.sh
new file mode 100755
index 0000000000..50728a137e
--- /dev/null
+++ b/scripts/ci/ci-driver.sh
@@ -0,0 +1,424 @@
+#!/bin/bash
+
+# This script is used to build Tor for continuous integration.  It should
+# be kept the same for all supported Tor versions.
+#
+# It's subject to the regular Tor license; see LICENSE for copying
+# information.
+
+set -o errexit
+set -o nounset
+
+# Options for this script.
+DEBUG_CI="${DEBUG_CI:-no}"
+COLOR_CI="${COLOR_CI:-yes}"
+
+# Options for which CI system this is.
+ON_GITLAB="${ON_GITLAB:-yes}"
+
+# Options for how to build Tor.  All should be yes/no.
+FATAL_WARNINGS="${FATAL_WARNINGS:-yes}"
+HARDENING="${HARDENING:-no}"
+COVERAGE="${COVERAGE:-no}"
+RUST="${RUST:-no}"
+DOXYGEN="${DOXYGEN:-no}"
+ASCIIDOC="${ASCIIDOC:-no}"
+
+# Options for which tests to run.   All should be yes/no.
+CHECK="${CHECK:-yes}"
+STEM="${STEM:-no}"
+CHUTNEY="${CHUTNEY:-no}"
+DISTCHECK="${DISTCHECK:-no}"
+
+# Options for where the Tor source is.
+CI_SRCDIR="${CI_SRCDIR:-.}"
+
+# Options for where to build.
+CI_BUILDDIR="${CI_BUILDDIR:-./build}"
+
+# How parallel should we run make?
+MAKE_J_OPT="${MAKE_J_OPT:--j4}"
+# Should we stop after make finds an error?
+MAKE_K_OPT="${MAKE_K_OPT:--k}"
+
+# What make target should we use for chutney?
+CHUTNEY_MAKE_TARGET="${CHUTNEY_MAKE_TARGET:-test-network}"
+
+# Where do we find our additional testing tools?
+CHUTNEY_PATH="${CHUTNEY_PATH:-}"
+STEM_PATH="${STEM_PATH:-}"
+
+#############################################################################
+# Preliminary functions.
+
+# Terminal coloring/emphasis stuff.
+if [[ "${COLOR_CI}" == "yes" ]]; then
+    T_RED=$(tput setaf 1 || true)
+    T_GREEN=$(tput setaf 2 || true)
+    T_DIM=$(tput dim || true)
+    T_BOLD=$(tput bold || true)
+    T_RESET=$(tput sgr0 || true)
+else
+    T_RED=
+    T_GREEN=
+    T_DIM=
+    T_BOLD=
+    T_RESET=
+fi
+
+function error()
+{
+    echo "${T_BOLD}${T_RED}ERROR:${T_RESET} $*" 1>&2
+}
+function die()
+{
+    echo "${T_BOLD}${T_RED}FATAL ERROR:${T_RESET} $*" 1>&2
+    exit 1
+}
+function hooray()
+{
+    echo "${T_BOLD}${T_GREEN}$*${T_RESET}"
+}
+
+if [[ "${DEBUG_CI}" == "yes" ]]; then
+    function debug()
+    {
+        echo "${T_DIM}(debug): $*${T_RESET}"
+    }
+else
+    function debug()
+    {
+        :
+    }
+fi
+
+function yes_or_no()
+{
+    local varname="$1"
+    local value="${!varname}"
+    debug "${varname} is ${value}"
+    if [[ "${value}" != 'yes' && "${value}" != 'no' ]]; then
+        die "${varname} must be 'yes' or 'no'.  Got unexpected value ${value}".
+    fi
+}
+
+function incompatible()
+{
+    local varname1="$1"
+    local varname2="$2"
+    local val1="${!varname1}"
+    local val2="${!varname2}"
+    if [[ "${val1}" = 'yes' && "${val2}" = 'yes' ]]; then
+        die "Cannot set both ${varname1} and ${varname2}: they are incompatible."
+    fi
+}
+
+function runcmd()
+{
+    echo "${T_BOLD}\$ $*${T_RESET}"
+    if ! "$@" ; then
+        error "command '$*' has failed."
+        return 1
+    fi
+}
+
+function show_git_version()
+{
+    local tool="$1"
+    local dir="$2"
+    local version="?????"
+    if [[ -e "$dir/.git" ]] ; then
+        version=$(cd "$dir"; git rev-parse HEAD)
+    fi
+    echo "${T_BOLD}$tool:${T_RESET} $version"
+}
+
+if [[ "${ON_GITLAB}" == "yes" ]]; then
+    function start_section()
+    {
+	local label="$1"
+	local stamp
+	stamp=$(date +%s)
+	printf "section_start:%s:%s\r\e[0K" "$stamp" "$label"
+	echo "${T_BOLD}${T_GREEN}========= $label${T_RESET}"
+    }
+    function end_section()
+    {
+	local label="$1"
+	local stamp
+	stamp=$(date +%s)
+	printf "section_end:%s:%s\r\e[0K" "$stamp" "$label"
+    }
+else
+    function start_section()
+    {
+	true
+    }
+    function end_section()
+    {
+	true
+    }
+fi
+
+if [[ "$*" == "" ]]; then
+    RUN_STAGE_CONFIGURE="yes"
+    RUN_STAGE_BUILD="yes"
+    RUN_STAGE_TEST="yes"
+else
+    RUN_STAGE_CONFIGURE="no"
+    RUN_STAGE_BUILD="no"
+    RUN_STAGE_TEST="no"
+
+    for stage in "$@"; do
+	case "$stage" in
+	    configure)
+		RUN_STAGE_CONFIGURE="yes"
+		;;
+	    build)
+		RUN_STAGE_BUILD="yes"
+		;;
+	    test)
+		RUN_STAGE_TEST="yes"
+		;;
+	    *)
+		error "Unknown stage $stage"
+		;;
+	esac
+    done
+fi
+
+#############################################################################
+# Validate inputs.
+
+debug Validating inputs
+yes_or_no DEBUG_CI
+yes_or_no COLOR_CI
+yes_or_no ON_GITLAB
+yes_or_no FATAL_WARNINGS
+yes_or_no HARDENING
+yes_or_no COVERAGE
+yes_or_no RUST
+yes_or_no DOXYGEN
+yes_or_no ASCIIDOC
+
+yes_or_no CHECK
+yes_or_no STEM
+yes_or_no DISTCHECK
+
+incompatible DISTCHECK CHECK
+incompatible DISTCHECK CHUTNEY
+incompatible DISTCHECK STEM
+incompatible DISTCHECK COVERAGE
+incompatible DISTCHECK DOXYGEN
+
+if [[ "${CHUTNEY}" = yes && "${CHUTNEY_PATH}" = '' ]] ; then
+    die "CHUTNEY is set to 'yes', but CHUTNEY_PATH was not specified."
+fi
+
+if [[ "${STEM}" = yes && "${STEM_PATH}" = '' ]] ; then
+    die "STEM is set to 'yes', but STEM_PATH was not specified."
+fi
+
+#############################################################################
+# Set up options for make and configure.
+
+make_options=()
+if [[ "$MAKE_J_OPT" != "" ]]; then
+    make_options+=("$MAKE_J_OPT")
+fi
+if [[ "$MAKE_K_OPT" != "" ]]; then
+    make_options+=("$MAKE_K_OPT")
+fi
+
+configure_options=()
+if [[ "$FATAL_WARNINGS" == "yes" ]]; then
+    configure_options+=("--enable-fatal-warnings")
+fi
+if [[ "$HARDENING" == "yes" ]]; then
+    configure_options+=("--enable-fragile-hardening")
+fi
+if [[ "$COVERAGE" == "yes" ]]; then
+    configure_options+=("--enable-coverage")
+fi
+if [[ "$RUST" == "yes" ]]; then
+    configure_options+=("--enable-rust")
+fi
+if [[ "$ASCIIDOC" != "yes" ]]; then
+    configure_options+=("--disable-asciidoc")
+fi
+
+#############################################################################
+# Tell the user about our versions of different tools and packages.
+
+uname -a
+python -V || echo "no 'python' binary."
+python3 -V || echo "no 'pythone' binary."
+
+show_git_version Tor "${CI_SRCDIR}"
+if [[ "${STEM}" = "yes" ]]; then
+    show_git_version Stem "${STEM_PATH}"
+fi
+if [[ "${CHUTNEY}" = "yes" ]]; then
+    show_git_version Chutney "${CHUTNEY_PATH}"
+fi
+
+#############################################################################
+# Make sure the directories are all there.
+
+# Make sure CI_SRCDIR exists and has a file we expect.
+if [[ ! -d "$CI_SRCDIR" ]] ; then
+    die "CI_SRCDIR=${CI_SRCDIR} is not a directory"
+fi
+if [[ ! -f "$CI_SRCDIR/src/core/or/or.h" ]] ; then
+    die "CI_SRCDIR=${CI_SRCDIR} does not look like a Tor directory."
+fi
+
+# Make CI_SRCDIR absolute.
+CI_SRCDIR=$(cd "$CI_SRCDIR" && pwd)
+
+# Create an "artifacts" directory to copy artifacts into.
+mkdir -p ./artifacts
+
+if [[ "$RUN_STAGE_CONFIGURE" = "yes" ]]; then
+
+    start_section "Autogen"
+    runcmd cd "${CI_SRCDIR}"
+    runcmd ./autogen.sh
+    runcmd mkdir -p "${CI_BUILDDIR}"
+    runcmd cd "${CI_BUILDDIR}"
+    end_section "Autogen"
+
+    # make the builddir absolute too.
+    CI_BUILDDIR=$(pwd)
+
+    start_section "Configure"
+    if ! runcmd "${CI_SRCDIR}"/configure "${configure_options[@]}" ; then
+	error "Here is the end of config.log:"
+	runcmd tail config.log
+	die "Unable to continue"
+    fi
+    end_section "Configure"
+else
+    debug "Skipping configure stage. Making sure that ${CI_BUILDDIR}/config.log exists."
+    if [[ ! -d "${CI_BUILDDIR}" ]]; then
+	die "Build directory ${CI_BUILDDIR} did not exist!";
+    fi
+    if [[ ! -f "${CI_BUILDDIR}/config.log" ]]; then
+	die "Tor was not configured in ${CI_BUILDDIR}!";
+    fi
+
+    cp config.log "${CI_SRCDIR}"/artifacts
+
+    runcmd cd "${CI_BUILDDIR}"
+    CI_BUILDDIR=$(pwd)
+fi
+
+###############################
+# Build Tor.
+
+if [[ "$RUN_STAGE_BUILD" = "yes" ]] ; then
+    if [[ "$DISTCHECK" = "no" ]]; then
+	start_section "Build"
+	runcmd make "${make_options[@]}" all
+        cp src/app/tor "${CI_SRCDIR}"/artifacts
+	end_section "Build"
+    else
+	export DISTCHECK_CONFIGURE_FLAGS="${configure_options[*]}"
+	# XXXX Set make options?
+	start_section Distcheck
+	if runcmd make "${make_options[@]}" distcheck ; then
+            hooray "Distcheck was successful. Nothing further will be done."
+            # We have to exit early here, since we can't do any other tests.
+            cp tor-*.tar.gz "${CI_SRCDIR}"/artifacts
+            exit 0
+	else
+            error "Diagnostics:"
+            runcmd make show-distdir-testlog || true
+            runcmd make show-distdir-core || true
+            die "Unable to continue."
+	fi
+	end_section Distcheck
+    fi
+fi
+##############################
+# Run tests.
+
+if [[ "$RUN_STAGE_TEST" == "no" ]]; then
+    echo "Skipping tests. Exiting now."
+    exit 0
+fi
+
+if [[ "$RUN_STAGE_BUILD" = "no" ]] ; then
+    debug "Skipped build stage. Making sure that ./src/app/tor exists."
+    if [[ ! -f "./src/app/tor" ]]; then
+	die "$(pwd)/src/app/tor does not exist"
+    fi
+fi
+
+FAILED_TESTS=""
+
+if [[ "${DOXYGEN}" = 'yes' ]]; then
+    start_section Doxygen
+    if runcmd make doxygen; then
+	hooray "make doxygen has succeeded."
+    else
+	FAILED_TESTS="${FAILED_TESTS} doxygen"
+    fi
+    end_section Doxygen
+fi
+
+if [[ "${CHECK}" = "yes" ]]; then
+    start_section "Check"
+    if runcmd make "${make_options[@]}" check; then
+        hooray "make check has succeeded."
+    else
+        error "Here are the contents of the test suite output:"
+        runcmd cat test_suite.log || true
+        FAILED_TESTS="${FAILED_TESTS} check"
+    fi
+    end_section "Check"
+fi
+
+if [[ "${CHUTNEY}" = "yes" ]]; then
+    start_section "Chutney"
+    if runcmd make "${CHUTNEY_MAKE_TARGET}"; then
+        hooray "Chutney tests have succeeded"
+    else
+        error "Chutney says:"
+        runcmd "${CHUTNEY_PATH}"/tools/diagnostics.sh || true
+        # XXXX These next two should be part of a make target.
+        runcmd ls test_network_log || true
+        runcmd cat test_network_log || true
+        FAILED_TESTS="${FAILED_TESTS} chutney"
+    fi
+    end_section "Chutney"
+fi
+
+if [[ "${STEM}" = "yes" ]]; then
+   start_section "Stem"
+   # XXXX This shold probably be part some test-stem make target.
+   if runcmd timelimit -p -t 520 -s USR1 -T 30 -S ABRT \
+         python3 "${STEM_PATH}/run_tests.py" \
+         --tor src/app/tor \
+         --integ --test control.controller \
+         --test control.base_controller \
+         --test process \
+         --log TRACE \
+         --log-file stem.log ; then
+       hooray "Stem tests have succeeded"
+   else
+       error "Stem output:"
+       runcmd tail -1000 "${STEM_PATH}"/test/data/tor_log
+       runcmd grep -v "SocketClosed" stem.log | tail -1000
+       FAILED_TESTS="${FAILED_TESTS} stem"
+   fi
+   end_section "Stem"
+fi
+
+# TODO: Coverage
+
+if [[ "${FAILED_TESTS}" != "" ]]; then
+    die "Failed tests: ${FAILED_TESTS}"
+fi
+
+hooray "Everything seems fine."
diff --git a/src/feature/rend/rendclient.c b/src/feature/rend/rendclient.c
index 2e119d7c99..1446dc2269 100644
--- a/src/feature/rend/rendclient.c
+++ b/src/feature/rend/rendclient.c
@@ -261,8 +261,8 @@ rend_client_send_introduction(origin_circuit_t *introcirc,
             > MAX_NICKNAME_LEN)) {
       goto perm_err;
     }
-    strncpy(tmp, rendcirc->build_state->chosen_exit->nickname,
-            (MAX_NICKNAME_LEN+1)); /* nul pads */
+    strlcpy(tmp, rendcirc->build_state->chosen_exit->nickname,
+            sizeof(tmp));
     memcpy(tmp+MAX_NICKNAME_LEN+1, rendcirc->rend_data->rend_cookie,
            REND_COOKIE_LEN);
     dh_offset = MAX_NICKNAME_LEN+1+REND_COOKIE_LEN;