diff options
Diffstat (limited to 'scripts')
31 files changed, 777 insertions, 241 deletions
diff --git a/scripts/README b/scripts/README index 9cd6e74ac7..4cb49370f5 100644 --- a/scripts/README +++ b/scripts/README @@ -12,7 +12,7 @@ never used. maint/checkOptionDocs.pl -- Make sure that Tor options are documented in the manpage, and that the manpage only documents real Tor options. -maint/checkSpaces.pl -- Style checker for the Tor source code. Mainly checks +maint/checkSpace.pl -- Style checker for the Tor source code. Mainly checks whitespace. maint/findMergedChanges.pl -- Find a set of changes/* files that have been @@ -23,7 +23,7 @@ maint/format_changelog.py -- Flow the changelog into the proper format. maint/redox.py -- Find places that should have DOCDOC comments to indicate a need for doxygen comments, and put those comments there. -maint/updateVersions.pl -- Update the version number in the .nsi and windows +maint/update_versions.py -- Update the version number in the .nsi and windows orconfig.h files. diff --git a/scripts/build/combine_libs b/scripts/build/combine_libs index 9c87f68248..9dec483602 100755 --- a/scripts/build/combine_libs +++ b/scripts/build/combine_libs @@ -8,14 +8,19 @@ ORIGDIR="$(pwd)" trap 'cd "$ORIGDIR" && rm -rf "$TMPDIR"' 0 abspath() { - echo "$(cd "$(dirname "$1")">/dev/null && pwd)/$(basename "$1")" + echo "$(cd "$(dirname "$1")" >/dev/null && pwd)/$(basename "$1")" } -TARGET=$(abspath "$1") +apple_symdef_fix() { + # On modern macOS and iOS we need to remove the "__.SYMDEF" and "__.SYMDEF + # SORTED" before we repack the archive. + # See: tor#40683. + if [ "$(uname -s)" = "Darwin" ] ; then + find . -name "__.SYMDEF*" -delete + fi +} -#echo ORIGDIR="$ORIGDIR" -#echo AR="$AR" -#echo ARFLAGS="$AFLAGS" +TARGET=$(abspath "$1") shift @@ -24,12 +29,12 @@ for input in "$@"; do abs=$(abspath "$input") dir="$TMPDIR"/$(basename "$input" .a) mkdir "$dir" - cd "$dir">/dev/null + cd "$dir" >/dev/null "${AR:-ar}" x "$abs" done cd "$TMPDIR" >/dev/null -#echo "${AR:-ar}" "${ARFLAGS:-cru}" library.tmp.a ./*/** +apple_symdef_fix "${AR:-ar}" "${ARFLAGS:-cru}" library.tmp.a ./*/** "${RANLIB:-ranlib}" library.tmp.a mv -f library.tmp.a "$TARGET" diff --git a/scripts/ci/ci-driver.sh b/scripts/ci/ci-driver.sh index 66c8f2ec26..ef31da1ca3 100755 --- a/scripts/ci/ci-driver.sh +++ b/scripts/ci/ci-driver.sh @@ -30,7 +30,6 @@ RUN_STAGE_TEST="${RUN_STAGE_TEST:-yes}" FATAL_WARNINGS="${FATAL_WARNINGS:-yes}" HARDENING="${HARDENING:-no}" COVERAGE="${COVERAGE:-no}" -RUST="${RUST:-no}" DOXYGEN="${DOXYGEN:-no}" ASCIIDOC="${ASCIIDOC:-no}" TRACING="${TRACING:-no}" @@ -87,6 +86,7 @@ function error() { echo "${T_BOLD}${T_RED}ERROR:${T_RESET} $*" 1>&2 } + function die() { echo "${T_BOLD}${T_RED}FATAL ERROR:${T_RESET} $*" 1>&2 @@ -159,27 +159,27 @@ function show_git_version() if [[ "${ON_GITLAB}" == "yes" ]]; then function start_section() { - local label="$1" - local stamp - stamp=$(date +%s) - printf "section_start:%s:%s\r\e[0K" "$stamp" "$label" - echo "${T_BOLD}${T_GREEN}========= $label${T_RESET}" + local label="$1" + local stamp + stamp=$(date +%s) + printf "section_start:%s:%s\r\e[0K" "$stamp" "$label" + echo "${T_BOLD}${T_GREEN}========= $label${T_RESET}" } function end_section() { - local label="$1" - local stamp - stamp=$(date +%s) - printf "section_end:%s:%s\r\e[0K" "$stamp" "$label" + local label="$1" + local stamp + stamp=$(date +%s) + printf "section_end:%s:%s\r\e[0K" "$stamp" "$label" } else function start_section() { - true + true } function end_section() { - true + true } fi @@ -193,7 +193,6 @@ yes_or_no ON_GITLAB yes_or_no FATAL_WARNINGS yes_or_no HARDENING yes_or_no COVERAGE -yes_or_no RUST yes_or_no DOXYGEN yes_or_no ASCIIDOC yes_or_no TRACING @@ -245,9 +244,6 @@ fi if [[ "$COVERAGE" == "yes" ]]; then configure_options+=("--enable-coverage") fi -if [[ "$RUST" == "yes" ]]; then - configure_options+=("--enable-rust") -fi if [[ "$ASCIIDOC" != "yes" ]]; then configure_options+=("--disable-asciidoc") fi @@ -297,19 +293,15 @@ TOR_VER_AT_LEAST_044=no # These are the currently supported Tor versions; no need to work with anything # ancient in this script. case "$TOR_VERSION" in - 0.3.*) - TOR_VER_AT_LEAST_043=no - TOR_VER_AT_LEAST_044=no - ;; - 0.4.[012].*) - TOR_VER_AT_LEAST_043=no - TOR_VER_AT_LEAST_044=no + 0.4.5.*) + TOR_VER_AT_LEAST_043=yes + TOR_VER_AT_LEAST_044=yes ;; - 0.4.3.*) + 0.4.6.*) TOR_VER_AT_LEAST_043=yes - TOR_VER_AT_LEAST_044=no + TOR_VER_AT_LEAST_044=yes ;; - *) + 0.4.7.*) TOR_VER_AT_LEAST_043=yes TOR_VER_AT_LEAST_044=yes ;; @@ -346,18 +338,18 @@ if [[ "$RUN_STAGE_CONFIGURE" = "yes" ]]; then start_section "Configure" if ! runcmd "${CI_SRCDIR}"/configure "${configure_options[@]}" ; then - error "Here is the end of config.log:" - runcmd tail config.log - die "Unable to continue" + error "Here is the end of config.log:" + runcmd tail config.log + die "Unable to continue" fi end_section "Configure" else debug "Skipping configure stage. Making sure that ${CI_BUILDDIR}/config.log exists." if [[ ! -d "${CI_BUILDDIR}" ]]; then - die "Build directory ${CI_BUILDDIR} did not exist!"; + die "Build directory ${CI_BUILDDIR} did not exist!" fi if [[ ! -f "${CI_BUILDDIR}/config.log" ]]; then - die "Tor was not configured in ${CI_BUILDDIR}!"; + die "Tor was not configured in ${CI_BUILDDIR}!" fi cp config.log "${CI_SRCDIR}"/artifacts @@ -371,26 +363,26 @@ fi if [[ "$RUN_STAGE_BUILD" = "yes" ]] ; then if [[ "$DISTCHECK" = "no" ]]; then - start_section "Build" - runcmd make "${make_options[@]}" all + start_section "Build" + runcmd make "${make_options[@]}" all cp src/app/tor "${CI_SRCDIR}"/artifacts - end_section "Build" + end_section "Build" else - export DISTCHECK_CONFIGURE_FLAGS="${configure_options[*]}" - # XXXX Set make options? - start_section Distcheck - if runcmd make "${make_options[@]}" distcheck ; then + export DISTCHECK_CONFIGURE_FLAGS="${configure_options[*]}" + # XXXX Set make options? + start_section Distcheck + if runcmd make "${make_options[@]}" distcheck ; then hooray "Distcheck was successful. Nothing further will be done." # We have to exit early here, since we can't do any other tests. cp tor-*.tar.gz "${CI_SRCDIR}"/artifacts exit 0 - else + else error "Diagnostics:" runcmd make show-distdir-testlog || true runcmd make show-distdir-core || true die "Unable to continue." - fi - end_section Distcheck + fi + end_section Distcheck fi fi @@ -408,9 +400,9 @@ if [[ "${DOXYGEN}" = 'yes' ]]; then start_section Doxygen if [[ "${TOR_VER_AT_LEAST_043}" = 'yes' ]]; then if runcmd make doxygen; then - hooray "make doxygen has succeeded." + hooray "make doxygen has succeeded." else - FAILED_TESTS="${FAILED_TESTS} doxygen" + FAILED_TESTS="${FAILED_TESTS} doxygen" fi else skipping "make doxygen: doxygen is broken for Tor < 0.4.3" @@ -444,6 +436,8 @@ if [[ "${CHUTNEY}" = "yes" ]]; then start_section "Chutney" export CHUTNEY_TOR_SANDBOX=0 export CHUTNEY_ALLOW_FAILURES=2 + # Send 5MB for every verify check. + export CHUTNEY_DATA_BYTES=5000000 if runcmd make "${CHUTNEY_MAKE_TARGET}"; then hooray "Chutney tests have succeeded" else @@ -452,43 +446,43 @@ if [[ "${CHUTNEY}" = "yes" ]]; then runcmd "${CHUTNEY_PATH}"/tools/diagnostics.sh || true # XXXX These next two should be part of a make target. runcmd ls test_network_log || true - runcmd cat test_network_log || true + runcmd head -n -0 test_network_log/* || true FAILED_TESTS="${FAILED_TESTS} chutney" fi end_section "Chutney" fi if [[ "${STEM}" = "yes" ]]; then - start_section "Stem" - # 0.3.5 and onward have now disabled onion service v2 so we need to exclude - # these Stem tests from now on. - EXCLUDE_TESTS="--exclude-test control.controller.test_ephemeral_hidden_services_v2 --exclude-test control.controller.test_hidden_services_conf --exclude-test control.controller.test_with_ephemeral_hidden_services_basic_auth --exclude-test control.controller.test_without_ephemeral_hidden_services --exclude-test control.controller.test_with_ephemeral_hidden_services_basic_auth_no_credentials" - if [[ "${TOR_VER_AT_LEAST_044}" = 'yes' ]]; then - # XXXX This should probably be part of some test-stem make target. - - # Disable the check around EXCLUDE_TESTS that requires double quote. We - # need it to be expanded. - # shellcheck disable=SC2086 - if runcmd timelimit -p -t 520 -s USR1 -T 30 -S ABRT \ - python3 "${STEM_PATH}/run_tests.py" \ - --tor src/app/tor \ - --integ --test control.controller \ - $EXCLUDE_TESTS \ - --test control.base_controller \ - --test process \ - --log TRACE \ - --log-file stem.log ; then - hooray "Stem tests have succeeded" - else - error "Stem output:" - runcmd tail -1000 "${STEM_PATH}"/test/data/tor_log - runcmd grep -v "SocketClosed" stem.log | tail -1000 - FAILED_TESTS="${FAILED_TESTS} stem" - fi - else - skipping "Stem: broken with <= 0.4.3. See bug tor#40077" - fi - end_section "Stem" + start_section "Stem" + # 0.3.5 and onward have now disabled onion service v2 so we need to exclude + # these Stem tests from now on. + EXCLUDE_TESTS="--exclude-test control.controller.test_ephemeral_hidden_services_v2 --exclude-test control.controller.test_hidden_services_conf --exclude-test control.controller.test_with_ephemeral_hidden_services_basic_auth --exclude-test control.controller.test_without_ephemeral_hidden_services --exclude-test control.controller.test_with_ephemeral_hidden_services_basic_auth_no_credentials" + if [[ "${TOR_VER_AT_LEAST_044}" = 'yes' ]]; then + # XXXX This should probably be part of some test-stem make target. + + # Disable the check around EXCLUDE_TESTS that requires double quote. We + # need it to be expanded. + # shellcheck disable=SC2086 + if runcmd timelimit -p -t 520 -s USR1 -T 30 -S ABRT \ + python3 "${STEM_PATH}/run_tests.py" \ + --tor src/app/tor \ + --integ --test control.controller \ + $EXCLUDE_TESTS \ + --test control.base_controller \ + --test process \ + --log TRACE \ + --log-file stem.log ; then + hooray "Stem tests have succeeded" + else + error "Stem output:" + runcmd tail -1000 "${STEM_PATH}"/test/data/tor_log + runcmd grep -v "SocketClosed" stem.log | tail -1000 + FAILED_TESTS="${FAILED_TESTS} stem" + fi + else + skipping "Stem: broken with <= 0.4.3. See bug tor#40077" + fi + end_section "Stem" fi # TODO: Coverage diff --git a/scripts/codegen/fuzzing_include_am.py b/scripts/codegen/fuzzing_include_am.py index b52b956f81..d5d5cb0154 100755 --- a/scripts/codegen/fuzzing_include_am.py +++ b/scripts/codegen/fuzzing_include_am.py @@ -6,16 +6,18 @@ from __future__ import print_function from __future__ import unicode_literals FUZZERS = """ + address + addressPTR consensus descriptor diff diff-apply extrainfo - hsdescv2 hsdescv3 + hsdescv3-inner + hsdescv3-middle http http-connect - iptsv2 microdesc socks strops @@ -32,7 +34,6 @@ FUZZING_LDFLAG = \ @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) @TOR_LDFLAGS_libevent@ FUZZING_LIBS = \ src/test/libtor-testing.a \ - $(rust_ldadd) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ @TOR_LIBEVENT_LIBS@ $(TOR_LIBS_CRYPTLIB) \ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ @CURVE25519_LIBS@ \ @@ -47,11 +48,10 @@ oss-fuzz-prereqs: \ noinst_HEADERS += \ src/test/fuzz/fuzzing.h -LIBFUZZER = -lFuzzer LIBFUZZER_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ LIBFUZZER_CFLAGS = $(FUZZING_CFLAGS) -LIBFUZZER_LDFLAG = $(FUZZING_LDFLAG) -LIBFUZZER_LIBS = $(FUZZING_LIBS) $(LIBFUZZER) -lstdc++ +LIBFUZZER_LDFLAG = $(FUZZING_LDFLAG) -fsanitize=fuzzer +LIBFUZZER_LIBS = $(FUZZING_LIBS) -lstdc++ LIBOSS_FUZZ_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ LIBOSS_FUZZ_CFLAGS = $(FUZZING_CFLAGS) diff --git a/scripts/git/git-list-tor-branches.sh b/scripts/git/git-list-tor-branches.sh index 5a527ffc05..dd3cf154b4 100755 --- a/scripts/git/git-list-tor-branches.sh +++ b/scripts/git/git-list-tor-branches.sh @@ -8,7 +8,7 @@ SCRIPT_NAME=$(basename "$0") function usage() { - echo "$SCRIPT_NAME [-h] [-l|-s|-b|-m] [-R]" + echo "$SCRIPT_NAME [-h] [-l|-s|-b|-m] [-R|-M]" echo echo " arguments:" echo " -h: show this help text" @@ -20,6 +20,7 @@ function usage() echo " ( branch parent path suffix parent_suffix ) arrays" echo echo " -R: omit release branches." + echo " -M: omit maint branches." } # list : just a list of branch names. @@ -27,9 +28,10 @@ function usage() # suffix: write a list of suffixes. # merge: branch, upstream, path, suffix, upstream suffix. mode="list" +skip_maint_branches="no" skip_release_branches="no" -while getopts "hblmsR" opt ; do +while getopts "hblmsRM" opt ; do case "$opt" in h) usage exit 0 @@ -42,6 +44,8 @@ while getopts "hblmsR" opt ; do ;; m) mode="merge" ;; + M) skip_maint_branches="yes" + ;; R) skip_release_branches="yes" ;; *) echo "Unknown option" @@ -73,13 +77,16 @@ branch() { # location: where the branch can be found. - if [[ "$brname" == "master" ]]; then - suffix="_master" + if [[ "$brname" == "main" ]]; then + suffix="_main" location="\$GIT_PATH/\$TOR_MASTER_NAME" elif [[ "$brname" =~ ^maint- ]]; then suffix="_${brname_nodots#maint-}" location="\$GIT_PATH/\$TOR_WKT_NAME/$brname" is_maint="yes" + if [[ "$skip_maint_branches" = "yes" ]]; then + return + fi elif [[ "$brname" =~ ^release- ]]; then suffix="_r${brname_nodots#release-}" location="\$GIT_PATH/\$TOR_WKT_NAME/$brname" @@ -136,15 +143,15 @@ finish() { # List of all branches. These must be in order, from oldest to newest, with # maint before release. -branch maint-0.3.5 -branch release-0.3.5 +branch maint-0.4.5 +branch release-0.4.5 -branch maint-0.4.3 -branch release-0.4.3 +branch maint-0.4.6 +branch release-0.4.6 -branch maint-0.4.4 -branch release-0.4.4 +branch maint-0.4.7 +branch release-0.4.7 -branch master +branch main finish diff --git a/scripts/git/git-merge-forward.sh b/scripts/git/git-merge-forward.sh index 7c72f8478d..d5d663d558 100755 --- a/scripts/git/git-merge-forward.sh +++ b/scripts/git/git-merge-forward.sh @@ -12,8 +12,8 @@ function usage() echo " (default: run commands)" echo " -t: test branch mode: create new branches from the commits checked" echo " out in each maint directory. Call these branches prefix_035," - echo " prefix_040, ... , prefix_master." - echo " (default: merge forward maint-*, release-*, and master)" + echo " prefix_040, ... , prefix_main." + echo " (default: merge forward maint-*, release-*, and main)" echo " -u: in test branch mode, if a prefix_* branch already exists," echo " skip creating that branch. Use after a merge error, to" echo " restart the merge forward at the first unmerged branch." @@ -28,7 +28,7 @@ function usage() echo echo " optional:" echo " TOR_MASTER: the name of the directory containing the tor.git clone" - echo " The tor master git directory is \$GIT_PATH/\$TOR_MASTER" + echo " The primary tor git directory is \$GIT_PATH/\$TOR_MASTER" echo " (default: tor; current: $TOR_MASTER_NAME)" echo " TOR_WKT_NAME: the name of the directory containing the tor" echo " worktrees. The tor worktrees are:" @@ -45,7 +45,7 @@ function usage() # Where are all those git repositories? GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"} -# The tor master git repository directory from which all the worktree have +# The main branch git repository directory from which all the worktree have # been created. TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"} # The worktrees location (directory). @@ -106,7 +106,7 @@ DRY_RUN=0 # Controlled by the -t <test-branch-prefix> option. The test branch base # name option makes git-merge-forward.sh create new test branches: -# <tbbn>_035, <tbbn>_040, ... , <tbbn>_master, and merge forward. +# <tbbn>_035, <tbbn>_040, ... , <tbbn>_main, and merge forward. TEST_BRANCH_PREFIX= # Controlled by the -u option. The use existing option checks for existing diff --git a/scripts/git/git-pull-all.sh b/scripts/git/git-pull-all.sh index 52a5c6140c..bbe2576d8e 100755 --- a/scripts/git/git-pull-all.sh +++ b/scripts/git/git-pull-all.sh @@ -20,7 +20,7 @@ usage() echo echo " optional:" echo " TOR_MASTER: the name of the directory containing the tor.git clone" - echo " The tor master git directory is \$GIT_PATH/\$TOR_MASTER" + echo " The primary tor git directory is \$GIT_PATH/\$TOR_MASTER" echo " (default: tor; current: $TOR_MASTER_NAME)" echo " TOR_WKT_NAME: the name of the directory containing the tor" echo " worktrees. The tor worktrees are:" @@ -37,7 +37,7 @@ usage() # Where are all those git repositories? GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"} -# The tor master git repository directory from which all the worktree have +# The primary tor git repository directory from which all the worktree have # been created. TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"} # The worktrees location (directory). @@ -51,7 +51,7 @@ set -e eval "$(git-list-tor-branches.sh -b)" set +e -# The master branch path has to be the main repository thus contains the +# The main branch path has to be the main repository thus contains the # origin that will be used to fetch the updates. All the worktrees are created # from that repository. ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME" @@ -159,20 +159,7 @@ function goto_repo function fetch_origin { local cmd="git fetch origin" - printf " %s Fetching origin..." "$MARKER" - if [ $DRY_RUN -eq 0 ]; then - msg=$( eval "$cmd" 2>&1 ) - validate_ret $? "$msg" - else - printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" - fi -} - -# Fetch tor-github pull requests. No arguments. -function fetch_tor_github -{ - local cmd="git fetch tor-github" - printf " %s Fetching tor-github..." "$MARKER" + printf "%s Fetching origin..." "$MARKER" if [ $DRY_RUN -eq 0 ]; then msg=$( eval "$cmd" 2>&1 ) validate_ret $? "$msg" @@ -185,7 +172,7 @@ function fetch_tor_github function fetch_tor_gitlab { local cmd="git fetch tor-gitlab" - printf " %s Fetching tor-gitlab..." "$MARKER" + printf "%s Fetching tor-gitlab..." "$MARKER" if [ $DRY_RUN -eq 0 ]; then msg=$( eval "$cmd" 2>&1 ) validate_ret $? "$msg" @@ -198,11 +185,10 @@ function fetch_tor_gitlab # Entry point # ############### -# First, fetch tor-github. +# Get into our origin repository. goto_repo "$ORIGIN_PATH" -fetch_tor_github -# Then tor-gitlab +# First, fetch tor-gitlab fetch_tor_gitlab # Then, fetch the origin. diff --git a/scripts/git/git-push-all.sh b/scripts/git/git-push-all.sh index 558ea8d01c..e5c16e615f 100755 --- a/scripts/git/git-push-all.sh +++ b/scripts/git/git-push-all.sh @@ -21,10 +21,10 @@ function usage() echo " -r: push to remote-name, rather than the default upstream remote." echo " (default: $DEFAULT_UPSTREAM_REMOTE, current: $UPSTREAM_REMOTE)" echo " -t: test branch mode: push test branches to remote-name. Pushes" - echo " branches prefix_035, prefix_040, ... , prefix_master." - echo " (default: push maint-*, release-*, and master)" + echo " branches prefix_035, prefix_040, ... , prefix_main." + echo " (default: push maint-*, release-*, and main)" echo " -s: push branches whose tips match upstream maint, release, or" - echo " master branches. The default is to skip these branches," + echo " main branches. The default is to skip these branches," echo " because they do not contain any new code. Use -s to test for" echo " CI environment failures, using code that previously passed CI." echo " (default: skip; current: $CURRENT_PUSH_SAME matching branches)" @@ -45,7 +45,7 @@ function usage() echo " (default: use the current directory for pushes;" echo " current: $TOR_FULL_GIT_PATH)" echo " TOR_MASTER: the name of the directory containing the tor.git clone" - echo " The tor master git directory is \$GIT_PATH/\$TOR_MASTER" + echo " The primary tor git directory is \$GIT_PATH/\$TOR_MASTER" echo " (default: tor; current: $TOR_MASTER_NAME)" echo echo " TOR_UPSTREAM_REMOTE_NAME: the default upstream remote." @@ -55,9 +55,9 @@ function usage() echo " Overridden by <git push options> after --." echo " (default: git push --atomic; current: $GIT_PUSH)" echo " TOR_PUSH_SAME: push branches whose tips match upstream maint," - echo " release, or master branches. Inverted by -s." + echo " release, or main branches. Inverted by -s." echo " (default: skip; current: $CURRENT_PUSH_SAME matching branches)" - echo " TOR_PUSH_DELAY: pushes the master and maint branches separately," + echo " TOR_PUSH_DELAY: pushes the main and maint branches separately," echo " so that CI runs in a sensible order." echo " (default: push all branches immediately; current: $PUSH_DELAY)" echo " we recommend that you set these env vars in your ~/.profile" @@ -71,7 +71,7 @@ set -e # Don't change this configuration - set the env vars in your .profile # -# The tor master git repository directory from which all the worktree have +# The primary tor git repository directory from which all the worktree have # been created. TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"} # Which directory do we push from? @@ -87,7 +87,7 @@ UPSTREAM_REMOTE=${DEFAULT_UPSTREAM_REMOTE} # Add a delay between pushes, so CI runs on the most important branches first PUSH_DELAY=${TOR_PUSH_DELAY:-0} # Push (1) or skip (0) test branches that are the same as an upstream -# maint/master branch. Push if you are testing that the CI environment still +# maint/main branch. Push if you are testing that the CI environment still # works on old code, skip if you are testing new code in the branch. # Default: skip unchanged branches. # Inverted by the -s option. @@ -99,7 +99,7 @@ PUSH_SAME=${TOR_PUSH_SAME:-0} # Controlled by the -t <test-branch-prefix> option. The test branch prefix # option makes git-merge-forward.sh create new test branches: -# <tbp>_035, <tbp>_040, ... , <tbp>_master, and merge each branch forward into +# <tbp>_035, <tbp>_040, ... , <tbp>_main, and merge each branch forward into # the next one. TEST_BRANCH_PREFIX= @@ -213,7 +213,7 @@ if [ "$TEST_BRANCH_PREFIX" ]; then # upstream branches (they have already been tested) UPSTREAM_SKIP_SAME_AS="$UPSTREAM_BRANCHES $DEFAULT_UPSTREAM_BRANCHES" else - # Skip the local maint-*, release-*, master branches that are the same as the + # Skip the local maint-*, release-*, main branches that are the same as the # current upstream branches, but ignore the default upstream # (we want to update a non-default remote, even if it matches the default) UPSTREAM_SKIP_SAME_AS="$UPSTREAM_BRANCHES" @@ -264,8 +264,8 @@ if [ "$PUSH_DELAY" -le 0 ]; then else # Push the branches in optimal CI order, with a delay between each push PUSH_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | sort -V) - MASTER_BRANCH=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep master) \ - || true # Skipped master branch + MASTER_BRANCH=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep main$) \ + || true # Skipped main branch if [ -z "$TEST_BRANCH_PREFIX" ]; then MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep maint) \ || true # Skipped all maint branches @@ -273,7 +273,7 @@ else tr "\\n" " ") || true # Skipped all release branches else # Actually test branches based on maint branches - MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep -v master) \ + MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep -v main$) \ || true # Skipped all maint test branches # No release branches RELEASE_BRANCHES= @@ -295,9 +295,9 @@ else # shellcheck disable=SC2086 for b in $MASTER_BRANCH $MAINT_BRANCHES; do $GIT_PUSH "$@" "$UPSTREAM_REMOTE" "$b" - # If we are pushing more than one branch, delay. - # In the unlikely scenario where we are pushing maint without master, - # or maint without release, there may be an extra delay + # If we are pushing more than one branch, delay. In the unlikely scenario + # where we are pushing maint branches without the main branch, or maint + # without release, there may be an extra delay if [ "$MAINT_BRANCHES" ] || [ "$RELEASE_BRANCHES" ]; then sleep "$PUSH_DELAY" fi diff --git a/scripts/git/git-setup-dirs.sh b/scripts/git/git-setup-dirs.sh index 5a9ae41cbd..c502f74f58 100755 --- a/scripts/git/git-setup-dirs.sh +++ b/scripts/git/git-setup-dirs.sh @@ -22,7 +22,7 @@ function usage() echo echo " optional:" echo " TOR_MASTER: the name of the directory containing the tor.git clone" - echo " The tor master git directory is \$GIT_PATH/\$TOR_MASTER" + echo " The primary tor git directory is \$GIT_PATH/\$TOR_MASTER" echo " (default: tor; current: $TOR_MASTER_NAME)" echo " TOR_WKT_NAME: the name of the directory containing the tor" echo " worktrees. The tor worktrees are:" @@ -65,7 +65,7 @@ function usage() # Where are all those git repositories? GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"} -# The tor master git repository directory from which all the worktree have +# The primary tor git repository directory from which all the worktree have # been created. TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"} # The worktrees location (directory). @@ -102,7 +102,7 @@ set -e eval "$(git-list-tor-branches.sh -b)" set +e -# The master branch path has to be the main repository thus contains the +# The main branch path has to be the main repository thus contains the # origin that will be used to fetch the updates. All the worktrees are created # from that repository. ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME" @@ -519,11 +519,11 @@ for ((i=0; i<COUNT; i++)); do repo_path=${!WORKTREE[$i]:1:1} printf "%s Handling branch %s\\n" "$MARKER" "${BYEL}$branch${CNRM}" - # We cloned the repository, and master is the default branch - if [ "$branch" = "master" ]; then - if [ "$TOR_MASTER_NAME" != "master" ]; then - # Set up a master link in the worktree directory - make_symlink "$repo_path" "$GIT_PATH/$TOR_WKT_NAME/master" + # We cloned the repository, and main is the default branch + if [ "$branch" = "main" ]; then + if [ "$TOR_MASTER_NAME" != "main" ]; then + # Set up a main branch link in the worktree directory + make_symlink "$repo_path" "$GIT_PATH/$TOR_WKT_NAME/main" fi else # git makes worktree directories if they don't exist diff --git a/scripts/git/post-merge.git-hook b/scripts/git/post-merge.git-hook index eae4f999e7..b458630d26 100755 --- a/scripts/git/post-merge.git-hook +++ b/scripts/git/post-merge.git-hook @@ -36,8 +36,8 @@ check_for_script_update() { } cur_branch=$(git rev-parse --abbrev-ref HEAD) -if [ "$cur_branch" != "master" ]; then - echo "post-merge: Not a master branch. Skipping." +if [ "$cur_branch" != "main" ]; then + echo "post-merge: Not a main branch. Skipping." exit 0 fi diff --git a/scripts/git/pre-push.git-hook b/scripts/git/pre-push.git-hook index f0a3a250ec..0f016df592 100755 --- a/scripts/git/pre-push.git-hook +++ b/scripts/git/pre-push.git-hook @@ -2,9 +2,9 @@ # git pre-push hook script to: # 0) Call the pre-commit hook, if it is available -# 1) prevent "fixup!" and "squash!" commit from ending up in master, release-* +# 1) prevent "fixup!" and "squash!" commit from ending up in main, release-* # or maint-* -# 2) Disallow pushing branches other than master, release-* +# 2) Disallow pushing branches other than main, release-* # and maint-* to origin (e.g. gitweb.torproject.org) # # To install this script, copy it into .git/hooks/pre-push path in your @@ -39,7 +39,7 @@ remote_name=$(git remote --verbose | grep "$2" | awk '{print $1}' | head -n 1) ref_is_upstream_branch() { - if [ "$1" == "refs/heads/master" ] || + if [ "$1" == "refs/heads/main" ] || [[ "$1" == refs/heads/release-* ]] || [[ "$1" == refs/heads/maint-* ]]; then return 1 @@ -54,8 +54,8 @@ do : else if [ "$remote_sha" = $z40 ]; then - # New branch, examine commits not in master - range="master...$local_sha" + # New branch, examine commits not in main + range="main...$local_sha" else # Update to existing branch, examine new commits range="$remote_sha..$local_sha" diff --git a/scripts/maint/checkOptionDocs.pl.in b/scripts/maint/checkOptionDocs.pl.in index bb8008c2e8..2d4a7884f5 100644 --- a/scripts/maint/checkOptionDocs.pl.in +++ b/scripts/maint/checkOptionDocs.pl.in @@ -43,7 +43,7 @@ open(F, "@abs_top_srcdir@/doc/man/tor.1.txt") or die; while (<F>) { if (m!^(?:\[\[([A-za-z0-9_]+)\]\] *)?\*\*([A-Za-z0-9_]+)\*\*!) { $manPageOptions{$2} = 1; - print "Missing an anchor: $2\n" unless (defined $1 or $2 eq 'tor'); + print "Missing an anchor: $2\n" unless (defined $1 or $2 eq 'tor'); } } close F; diff --git a/scripts/maint/checkShellScripts.sh b/scripts/maint/checkShellScripts.sh index 0a423be29e..f7e260bbd2 100755 --- a/scripts/maint/checkShellScripts.sh +++ b/scripts/maint/checkShellScripts.sh @@ -41,7 +41,6 @@ rm -f "$TOPLEVEL/contrib/dist/suse/tor.sh" "$TOPLEVEL/contrib/dist/tor.sh" find "$TOPLEVEL/contrib" "$TOPLEVEL/doc" "$TOPLEVEL/scripts" "$TOPLEVEL/src" \ -name "*.sh" \ -not -path "$TOPLEVEL/src/ext/*" \ - -not -path "$TOPLEVEL/src/rust/registry/*" \ -exec shellcheck {} + # Check scripts that aren't named *.sh diff --git a/scripts/maint/checkSpace.pl b/scripts/maint/checkSpace.pl index 857ce6f6f1..8ecbf414cf 100755 --- a/scripts/maint/checkSpace.pl +++ b/scripts/maint/checkSpace.pl @@ -7,13 +7,13 @@ my $found = 0; my $COLON_POS = 10; sub msg { - $found = 1; - my $v = shift; - $v =~ /^\s*([^:]+):(.*)$/; - chomp(my $errtype = $1); - my $rest = $2; - my $padding = ' ' x ($COLON_POS - length $errtype); - print "$padding$errtype:$rest\n"; + $found = 1; + my $v = shift; + $v =~ /^\s*([^:]+):(.*)$/; + chomp(my $errtype = $1); + my $rest = $2; + my $padding = ' ' x ($COLON_POS - length $errtype); + print "$padding$errtype:$rest\n"; } my $C = 0; diff --git a/scripts/maint/check_config_macros.pl b/scripts/maint/check_config_macros.pl index bcde2beccc..1398b9984a 100755 --- a/scripts/maint/check_config_macros.pl +++ b/scripts/maint/check_config_macros.pl @@ -7,7 +7,7 @@ my @macros = (); open(F, 'orconfig.h.in'); while(<F>) { if (/^#undef +([A-Za-z0-9_]*)/) { - push @macros, $1; + push @macros, $1; } } close F; @@ -15,6 +15,6 @@ close F; for my $m (@macros) { my $s = `git grep '$m' src`; if ($s eq '') { - print "Unused: $m\n"; + print "Unused: $m\n"; } } diff --git a/scripts/maint/findMergedChanges.pl b/scripts/maint/findMergedChanges.pl index d6c4105b74..427f2b111d 100755 --- a/scripts/maint/findMergedChanges.pl +++ b/scripts/maint/findMergedChanges.pl @@ -9,7 +9,7 @@ sub nChanges { # requires perl 5.8. Avoids shell issues if we ever get a changes # file named by the parents of Little Johnny Tables. open F, "-|", "git", "log", "--no-merges", "--pretty=format:%H", $branches, "--", $fname - or die "$!"; + or die "$!"; my @changes = <F>; return scalar @changes } @@ -22,13 +22,13 @@ Usage: findMergedChanges.pl [--merged/--unmerged/--weird/--list] [--branch=<branchname] [--head=<branchname>] changes/* A change is "merged" if it has ever been merged to release-0.2.4 and it has had -no subsequent changes in master. +no subsequent changes in main. A change is "unmerged" if it has never been merged to release-0.2.4 and it -has had changes in master. +has had changes in main. A change is "weird" if it has been merged to release-0.2.4 and it *has* had -subsequent changes in master. +subsequent changes in main. Suggested application: findMergedChanges.pl --merged changes/* | xargs -n 1 git rm @@ -37,18 +37,18 @@ EOF } my $target_branch = "origin/release-0.2.4"; -my $head = "origin/master"; +my $head = "origin/main"; while (@ARGV and $ARGV[0] =~ /^--/) { my $flag = shift @ARGV; if ($flag =~ /^--(weird|merged|unmerged|list)/) { - $look_for_type = $1; + $look_for_type = $1; } elsif ($flag =~ /^--branch=(\S+)/) { $target_branch = $1; } elsif ($flag =~ /^--head=(\S+)/) { $head = $1; } else { - die "Unrecognized flag $flag"; + die "Unrecognized flag $flag"; } } @@ -58,16 +58,16 @@ for my $changefile (@ARGV) { my $type; if ($n_merged != 0 and $n_postmerged == 0) { - $type = "merged"; + $type = "merged"; } elsif ($n_merged == 0 and $n_postmerged != 0) { - $type = "unmerged"; + $type = "unmerged"; } else { - $type = "weird"; + $type = "weird"; } if ($type eq $look_for_type) { - print "$changefile\n"; + print "$changefile\n"; } elsif ($look_for_type eq 'list') { - printf "% 8s: %s\n", $type, $changefile; + printf "% 8s: %s\n", $type, $changefile; } } diff --git a/scripts/maint/format_changelog.py b/scripts/maint/format_changelog.py index 93ab56e257..32b47ffcbb 100755 --- a/scripts/maint/format_changelog.py +++ b/scripts/maint/format_changelog.py @@ -424,7 +424,7 @@ def bug_html(m): try: disp_prefix, url_prefix = ISSUE_PREFIX_MAP[prefix] except KeyError: - print("Can't figure out URL for {}{}".formt(prefix,bugno), + print("Can't figure out URL for {}{}".format(prefix,bugno), file=sys.stderr) return "{} {}{}".format(kind, prefix, bugno) diff --git a/scripts/maint/gen_ccls_file.sh b/scripts/maint/gen_ccls_file.sh index 899e4e9603..04e31d22a8 100755 --- a/scripts/maint/gen_ccls_file.sh +++ b/scripts/maint/gen_ccls_file.sh @@ -15,6 +15,17 @@ CCLS_FILE=".ccls" PRIVATE_DEFS=$(grep -r --include \*.h "_PRIVATE" | grep "#ifdef" | cut -d' ' -f2 | sort | uniq) echo "clang" > "$CCLS_FILE" + +# Add these include so the ccls server can properly check new files that are +# not in the compile_commands.json yet +{ + echo "-I." + echo "-I./src" + echo "-I./src/ext" + echo "-I./src/ext/trunnel" +} >> "$CCLS_FILE" + +# Add all defines (-D). for p in $PRIVATE_DEFS; do - echo "-D$p" >> "$CCLS_FILE" + echo "-D$p" >> "$CCLS_FILE" done diff --git a/scripts/maint/geoip/README.geoip b/scripts/maint/geoip/README.geoip new file mode 100644 index 0000000000..0ed94b2276 --- /dev/null +++ b/scripts/maint/geoip/README.geoip @@ -0,0 +1,25 @@ +To generate new geoip files, you'll need to install the +libloc/"location" tool provided by https://location.ipfire.org/. +I personally build it with: + + ./configure CFLAGS='-g -O2' --disable-perl --without-systemd --prefix=/opt/libloc + make + make install + +Then (after adjusting PATH and PYTHONPATH) you can get the latest +dump with: + + location update + location dump geoip-dump.txt + +And transform it into geoip files with + + cargo run --release -- -i geoip-dump.txt + + +============================== + +Note that the current version "0.1.9" of rangemap has a performance +bug, making this tool quite slow. Previous versions had a +correctness bug that made the output needlessly long. With luck, +there will soon be a fast correct rangemap version.
\ No newline at end of file diff --git a/scripts/maint/geoip/geoip-db-tool/.gitignore b/scripts/maint/geoip/geoip-db-tool/.gitignore new file mode 100644 index 0000000000..eb5a316cbd --- /dev/null +++ b/scripts/maint/geoip/geoip-db-tool/.gitignore @@ -0,0 +1 @@ +target diff --git a/scripts/maint/geoip/geoip-db-tool/Cargo.lock b/scripts/maint/geoip/geoip-db-tool/Cargo.lock new file mode 100644 index 0000000000..ba610d4fc3 --- /dev/null +++ b/scripts/maint/geoip/geoip-db-tool/Cargo.lock @@ -0,0 +1,110 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "argh" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91792f088f87cdc7a2cfb1d617fa5ea18d7f1dc22ef0e1b5f82f3157cdc522be" +dependencies = [ + "argh_derive", + "argh_shared", +] + +[[package]] +name = "argh_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4eb0c0c120ad477412dc95a4ce31e38f2113e46bd13511253f79196ca68b067" +dependencies = [ + "argh_shared", + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "argh_shared" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "781f336cc9826dbaddb9754cb5db61e64cab4f69668bd19dcc4a0394a86f4cb1" + +[[package]] +name = "geoip-db-tool" +version = "0.1.0" +dependencies = [ + "argh", + "ipnetwork", + "rangemap", +] + +[[package]] +name = "heck" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "ipnetwork" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02c3eaab3ac0ede60ffa41add21970a7df7d91772c03383aac6c2c3d53cc716b" +dependencies = [ + "serde", +] + +[[package]] +name = "proc-macro2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rangemap" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90531bef860f96441c4cb74a1e43c281cd1366143928f944546ef0b1c60392b0" + +[[package]] +name = "serde" +version = "1.0.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d5161132722baa40d802cc70b15262b98258453e85e5d1d365c757c73869ae" + +[[package]] +name = "syn" +version = "1.0.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "unicode-segmentation" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796" + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" diff --git a/scripts/maint/geoip/geoip-db-tool/Cargo.toml b/scripts/maint/geoip/geoip-db-tool/Cargo.toml new file mode 100644 index 0000000000..8c794ca058 --- /dev/null +++ b/scripts/maint/geoip/geoip-db-tool/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "geoip-db-tool" +version = "0.1.0" +authors = ["Nick Mathewson <nickm@torproject.org>"] +edition = "2018" +license = "MIT OR Apache-2.0" +publish = false + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +ipnetwork= "0.17.0" +rangemap = "0.1.10" +argh = "0.1.4" diff --git a/scripts/maint/geoip/geoip-db-tool/src/db.rs b/scripts/maint/geoip/geoip-db-tool/src/db.rs new file mode 100644 index 0000000000..316182d823 --- /dev/null +++ b/scripts/maint/geoip/geoip-db-tool/src/db.rs @@ -0,0 +1,138 @@ +/// Code to parse a dump file +use std::collections::HashMap; +use std::convert::TryInto; +use std::iter::Peekable; + +use super::{AsBlock, NetBlock}; + +pub struct BlockReader<I> +where + I: Iterator<Item = std::io::Result<String>>, +{ + iter: Peekable<I>, +} + +pub enum AnyBlock { + NetBlock(NetBlock), + AsBlock(AsBlock), + OtherBlock, +} + +impl<I> BlockReader<I> +where + I: Iterator<Item = std::io::Result<String>>, +{ + pub fn new(iter: I) -> Self { + BlockReader { + iter: iter.peekable(), + } + } + + /// Extract the initial header from the file. + pub fn extract_header(&mut self) -> String { + let mut res: String = "".to_string(); + + while let Some(Ok(line)) = self.iter.peek() { + if !line.starts_with('#') { + break; + } + res.push_str(line.as_str()); + res.push('\n'); + let _ = self.iter.next(); + } + + res + } + + /// Extract the next empty-line-delimited block from the file. + /// + /// This isn't terribly efficient, but it's "fast enough". + fn get_block(&mut self) -> Option<std::io::Result<AnyBlock>> { + let mut kv = HashMap::new(); + + while let Some(line) = self.iter.next() { + //dbg!(&line); + if let Err(e) = line { + return Some(Err(e)); + } + let line_orig = line.unwrap(); + let line = line_orig.splitn(2, '#').next().unwrap().trim(); + if line.is_empty() { + if kv.is_empty() { + continue; + } else { + break; + } + } + let kwds: Vec<_> = line.splitn(2, ':').collect(); + if kwds.len() != 2 { + return None; // XXXX handle the error better. + } + kv.insert(kwds[0].trim().to_string(), kwds[1].trim().to_string()); + } + + if kv.is_empty() { + return None; + } + + if let Some(name) = kv.remove("name") { + // This is an AS block. + let asn = kv.get("aut-num").unwrap(); // XXXX handle error better + assert!(asn.starts_with("AS")); + let asn = asn[2..].parse().unwrap(); + return Some(Ok(AnyBlock::AsBlock(AsBlock { name, asn }))); + } + + let net = if let Some(net) = kv.get("net") { + net.parse().unwrap() //XXXX handle the error better. + } else { + return Some(Ok(AnyBlock::OtherBlock)); + }; + + let asn = if let Some(asn) = kv.get("aut-num") { + asn.parse().ok() + } else { + None + }; + + let cc = if let Some(country) = kv.get("country") { + assert!(country.as_bytes().len() == 2); + country.as_bytes()[0..2].try_into().unwrap() + } else { + *b"??" + }; + + fn is_true(v: Option<&String>) -> bool { + match v { + Some(s) => s == "true", + None => false, + } + } + + let is_anon_proxy = is_true(kv.get("is-anonymous-proxy")); + let is_anycast = is_true(kv.get("is-anycast-proxy")); + let is_satellite = is_true(kv.get("is-satellite-provider")); + + Some(Ok(AnyBlock::NetBlock(NetBlock { + net, + asn, + cc, + is_anon_proxy, + is_anycast, + is_satellite, + }))) + } +} + +impl<I> Iterator for BlockReader<I> +where + I: Iterator<Item = std::io::Result<String>>, +{ + type Item = AnyBlock; + fn next(&mut self) -> Option<Self::Item> { + match self.get_block() { + Some(Ok(b)) => Some(b), + _ => None, + } + } +} diff --git a/scripts/maint/geoip/geoip-db-tool/src/main.rs b/scripts/maint/geoip/geoip-db-tool/src/main.rs new file mode 100644 index 0000000000..9a22598a35 --- /dev/null +++ b/scripts/maint/geoip/geoip-db-tool/src/main.rs @@ -0,0 +1,239 @@ +/// A basic tool to convert IPFire Location dumps into the CSV formats that Tor +/// expects. +mod db; + +use argh::FromArgs; +use ipnetwork::IpNetwork; +use rangemap::RangeInclusiveMap; + +use std::fs::File; +use std::io::{BufRead, BufReader, BufWriter, Write}; +use std::net::{IpAddr, Ipv6Addr}; +use std::num::NonZeroU32; +use std::path::PathBuf; + +fn default_ipv4_path() -> PathBuf { + "./geoip".into() +} +fn default_ipv6_path() -> PathBuf { + "./geoip6".into() +} + +#[derive(FromArgs)] +/// Convert an IPFire Location dump into CSV geoip files. +struct Args { + /// where to store the IPv4 geoip output + #[argh(option, default = "default_ipv4_path()", short = '4')] + output_ipv4: PathBuf, + + /// where to store the IPv6 geoip6 output + #[argh(option, default = "default_ipv6_path()", short = '6')] + output_ipv6: PathBuf, + + /// where to find the dump file + #[argh(option, short = 'i')] + input: PathBuf, + + /// whether to include AS information in our output + #[argh(switch)] + include_asn: bool, + + /// where to store the AS map. + #[argh(option)] + output_asn: Option<PathBuf>, +} + +/// Represents a network block from running `location dump`. +#[derive(Debug, Clone)] +pub struct NetBlock { + pub net: IpNetwork, + pub cc: [u8; 2], + pub asn: Option<NonZeroU32>, + pub is_anon_proxy: bool, + pub is_anycast: bool, + pub is_satellite: bool, +} + +/// Represents an AS definition from running `location dump`. +#[derive(Debug, Clone, Ord, PartialOrd, Eq, PartialEq)] +pub struct AsBlock { + pub asn: NonZeroU32, + pub name: String, +} + +impl PartialEq for NetBlock { + fn eq(&self, other: &Self) -> bool { + self.net == other.net + } +} + +/// We define network blocks as being sorted first from largest to smallest, +/// then by address. +impl Ord for NetBlock { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.net + .prefix() + .cmp(&other.net.prefix()) + .then_with(|| self.net.network().cmp(&other.net.network())) + } +} + +impl PartialOrd for NetBlock { + fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> { + Some(self.cmp(other)) + } +} + +impl Eq for NetBlock {} + +#[derive(Copy, Clone, Eq, PartialEq, Debug)] +struct NetDefn { + cc: [u8; 2], + asn: Option<NonZeroU32>, +} + +impl NetBlock { + fn into_defn(self, include_asn: bool) -> NetDefn { + if include_asn { + NetDefn { + cc: self.cc, + asn: self.asn, + } + } else { + NetDefn { + cc: self.cc, + asn: None, + } + } + } +} + +impl NetDefn { + fn cc(&self) -> &str { + std::str::from_utf8(&self.cc).unwrap() + } + fn asn(&self) -> u32 { + match self.asn { + Some(v) => v.into(), + None => 0, + } + } +} + +const PROLOGUE: &str = "\ +# This file has been converted from the IPFire Location database +# using Tor's geoip-db-tool. For more information on the data, see +# https://location.ipfire.org/. +# +# Below is the header from the original export: +# +"; + +/// Read an input file in the `location dump` format, and write CSV ipv4 and ipv6 files. +/// +/// This code tries to be "efficient enough"; most of the logic is handled by +/// using the rangemap crate. +fn convert(args: Args) -> std::io::Result<()> { + let input = args.input.as_path(); + let output_v4 = args.output_ipv4.as_path(); + let output_v6 = args.output_ipv6.as_path(); + let include_asn = args.include_asn; + + let f = File::open(input)?; + let f = BufReader::new(f); + let mut blocks = Vec::new(); + let mut networks = Vec::new(); + + let mut reader = db::BlockReader::new(f.lines()); + let hdr = reader.extract_header(); + // Read blocks, and then sort them by specificity and address. + for nb in reader { + match nb { + db::AnyBlock::AsBlock(a) => networks.push(a), + db::AnyBlock::NetBlock(n) => blocks.push(n), + _ => {} + } + } + blocks.sort(); + + // Convert the sorted blocks into a map from address ranges into + // country codes. + // + // Note that since we have sorted the blocks from least to most specific, + // we will be puttting them into the maps in the right order, so that the + // most specific rule "wins". + // + // We use u32 and u128 as the index types for these RangeInclusiveMaps, + // so that we don't need to implement a step function for IpAddr. + let mut v4map: RangeInclusiveMap<u32, NetDefn, _> = RangeInclusiveMap::new(); + let mut v6map: RangeInclusiveMap<u128, NetDefn, _> = RangeInclusiveMap::new(); + + let mut n = 0usize; + let num_blocks = blocks.len(); + for nb in blocks { + n += 1; + if n % 100000 == 0 { + println!("{}/{}", n, num_blocks); + } + let start = nb.net.network(); + let end = nb.net.broadcast(); + match (start, end) { + (IpAddr::V4(a), IpAddr::V4(b)) => { + v4map.insert(a.into()..=b.into(), nb.into_defn(include_asn)); + } + (IpAddr::V6(a), IpAddr::V6(b)) => { + v6map.insert(a.into()..=b.into(), nb.into_defn(include_asn)); + } + (_, _) => panic!("network started and ended in different families!?"), + } + } + + // Write the ranges out to the appropriate files, in order. + let mut v4 = BufWriter::new(File::create(output_v4)?); + let mut v6 = BufWriter::new(File::create(output_v6)?); + + v4.write_all(PROLOGUE.as_bytes())?; + v4.write_all(hdr.as_bytes())?; + for (r, defn) in v4map.iter() { + let a: u32 = *r.start(); + let b: u32 = *r.end(); + if include_asn { + writeln!(&mut v4, "{},{},{},{}", a, b, defn.cc(), defn.asn())?; + } else { + writeln!(&mut v4, "{},{},{}", a, b, defn.cc())?; + } + } + + v6.write_all(PROLOGUE.as_bytes())?; + v6.write_all(hdr.as_bytes())?; + for (r, defn) in v6map.iter() { + let a: Ipv6Addr = (*r.start()).into(); + let b: Ipv6Addr = (*r.end()).into(); + if include_asn { + writeln!(&mut v6, "{},{},{},{}", a, b, defn.cc(), defn.asn())?; + } else { + writeln!(&mut v6, "{},{},{}", a, b, defn.cc())?; + } + } + + // The documentation says you should always flush a BufWriter. + v4.flush()?; + v6.flush()?; + + if let Some(output_asn) = args.output_asn { + networks.sort(); + let mut asn = BufWriter::new(File::create(output_asn)?); + for net in networks { + writeln!(&mut asn, "{},{}", net.asn, net.name)?; + } + asn.flush()?; + } + + Ok(()) +} + +fn main() -> std::io::Result<()> { + let args: Args = argh::from_env(); + + convert(args) +} diff --git a/scripts/maint/geoip/update_and_commit_geoip.sh b/scripts/maint/geoip/update_and_commit_geoip.sh new file mode 100755 index 0000000000..a83f82433e --- /dev/null +++ b/scripts/maint/geoip/update_and_commit_geoip.sh @@ -0,0 +1,34 @@ +#!/bin/sh + +set -e + +CMDDIR=$(cd "$(dirname "$0")" && pwd) + +if [ ! -e "./src/config/geoip" ] ; then + echo "Run this from inside the root dir of your oldest LTS repository" + exit 1 +fi + +if [ -n "$(git status --untracked-files=no --porcelain)" ]; then + echo "Working directory is not clean." + exit 1 +fi + +TOPDIR=$(pwd) +cd "./src/config/" +"${CMDDIR}/update_geoip.sh" +cd "${TOPDIR}" + +DASH_DATE=$(date -u +"%Y-%m-%d") +SLASH_DATE=$(date -u +"%Y/%m/%d") +CHANGESFILE="changes/geoip-$DASH_DATE" + +cat > "$CHANGESFILE" <<EOF + o Minor features (geoip data): + - Update the geoip files to match the IPFire Location Database, + as retrieved on ${SLASH_DATE}. +EOF + +git add "$CHANGESFILE" + +git commit -a -m "Update geoip files to match ipfire location db, $SLASH_DATE." diff --git a/scripts/maint/geoip/update_geoip.sh b/scripts/maint/geoip/update_geoip.sh new file mode 100755 index 0000000000..743683ab62 --- /dev/null +++ b/scripts/maint/geoip/update_geoip.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +set -e + +DIR=$(cd "$(dirname "$0")" && pwd) +TMP=$(mktemp -d) + +location --quiet update +location dump "$TMP/geoip-dump.txt" + +OLDDIR=$(pwd) +cd "$DIR/geoip-db-tool/" +cargo build --release +cd "$OLDDIR" + +"$DIR/geoip-db-tool/target/release/geoip-db-tool" -i "$TMP/geoip-dump.txt" diff --git a/scripts/maint/practracker/includes.py b/scripts/maint/practracker/includes.py index a5ee728824..46630d987f 100755 --- a/scripts/maint/practracker/includes.py +++ b/scripts/maint/practracker/includes.py @@ -40,11 +40,13 @@ def warn(msg): print(msg, file=sys.stderr) def fname_is_c(fname): - """ Return true iff 'fname' is the name of a file that we should - search for possibly disallowed #include directives. """ - if fname.endswith(".h") or fname.endswith(".c"): + """ + Return true if 'fname' is the name of a file that we should + search for possibly disallowed #include directives. + """ + if fname.endswith((".c", ".h")): bname = os.path.basename(fname) - return not (bname.startswith(".") or bname.startswith("#")) + return not bname.startswith((".", "#")) else: return False diff --git a/scripts/maint/practracker/test_practracker.sh b/scripts/maint/practracker/test_practracker.sh index e29b9106de..bb734ad9cd 100755 --- a/scripts/maint/practracker/test_practracker.sh +++ b/scripts/maint/practracker/test_practracker.sh @@ -1,15 +1,15 @@ #!/bin/sh -# Fail if any subprocess fails unexpectedly +# Fail this script if any subprocess fails unexpectedly. set -e umask 077 unset TOR_DISABLE_PRACTRACKER TMPDIR="" -clean () { +clean() { if [ -n "$TMPDIR" ] && [ -d "$TMPDIR" ]; then - rm -rf "$TMPDIR" + rm -rf "$TMPDIR" fi } trap clean EXIT HUP INT TERM @@ -27,7 +27,7 @@ PRACTRACKER_DIR="scripts/maint/practracker" TMPDIR="$(mktemp -d -t pracktracker.test.XXXXXX)" if test -z "${TMPDIR}" || test ! -d "${TMPDIR}" ; then echo >&2 "mktemp failed." - exit 1; + exit 1 fi DATA="${PRACTRACKER_DIR}/testdata" @@ -43,6 +43,7 @@ run_practracker() { --terse \ "${DATA}/" "$@" || echo "practracker exit status: $?" } + compare() { # we can't use cmp because we need to use -b for windows diff -b -u "$@" > "${TMPDIR}/test-diff" || true diff --git a/scripts/maint/practracker/util.py b/scripts/maint/practracker/util.py index c52ca2fbbf..6ab10a8de7 100644 --- a/scripts/maint/practracker/util.py +++ b/scripts/maint/practracker/util.py @@ -7,8 +7,7 @@ import os # We don't want to run metrics for unittests, automatically-generated C files, # external libraries or git leftovers. -EXCLUDE_SOURCE_DIRS = {"src/test/", "src/trunnel/", "src/rust/", - "src/ext/" } +EXCLUDE_SOURCE_DIRS = {"src/test/", "src/trunnel/", "src/ext/" } EXCLUDE_FILES = {"orconfig.h"} diff --git a/scripts/maint/run_calltool.sh b/scripts/maint/run_calltool.sh index b0268322f4..025a49cd03 100755 --- a/scripts/maint/run_calltool.sh +++ b/scripts/maint/run_calltool.sh @@ -5,8 +5,8 @@ set -e if test "x$CALLTOOL_PATH" != "x"; then - PYTHONPATH="${CALLTOOL_PATH}:${PYTHONPATH}" - export PYTHONPATH + PYTHONPATH="${CALLTOOL_PATH}:${PYTHONPATH}" + export PYTHONPATH fi mkdir -p callgraph @@ -14,8 +14,8 @@ mkdir -p callgraph SUBITEMS="fn_graph fn_invgraph fn_scc fn_scc_weaklinks module_graph module_invgraph module_scc module_scc_weaklinks" for calculation in $SUBITEMS; do - echo "======== $calculation" - python -m calltool "$calculation" > callgraph/"$calculation" + echo "======== $calculation" + python -m calltool "$calculation" > callgraph/"$calculation" done cat <<EOF > callgraph/README diff --git a/scripts/maint/updateRustDependencies.sh b/scripts/maint/updateRustDependencies.sh deleted file mode 100755 index 6d0587351f..0000000000 --- a/scripts/maint/updateRustDependencies.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018 The Tor Project, Inc. -# Copyright (c) 2018 isis agora lovecruft -# See LICENSE for license information -# -# updateRustDependencies.sh -# ------------------------- -# Update our vendored Rust dependencies, either adding/removing -# dependencies and/or upgrading current dependencies to newer -# versions. -# -# To use this script, first add your dependencies, exactly specifying -# their versions, into the appropriate *crate-level* Cargo.toml in -# src/rust/ (i.e. *not* /src/rust/Cargo.toml, but instead the one for -# your crate). -# -# Next, run this script. Then, go into src/ext/rust and commit the -# changes to the tor-rust-dependencies repo. - -set -e - -HERE=$(dirname "$(realpath "$0")") -TOPLEVEL=$(dirname "$(dirname "$HERE")") -TOML="$TOPLEVEL/src/rust/Cargo.toml" -VENDORED="$TOPLEVEL/src/ext/rust/crates" -CARGO=$(command -v cargo) - -if ! test -f "$TOML" ; then - printf "Error: Couldn't find workspace Cargo.toml in expected location: %s\\n" "$TOML" -fi - -if ! test -d "$VENDORED" ; then - printf "Error: Couldn't find directory for Rust dependencies! Expected location: %s\\n" "$VENDORED" -fi - -if test -z "$CARGO" ; then - printf "Error: cargo must be installed and in your \$PATH\\n" -fi - -if test -z "$(cargo --list | grep vendor)" ; then - printf "Error: cargo-vendor not installed\\n" -fi - -$CARGO vendor -v --locked --explicit-version --no-delete --sync "$TOML" "$VENDORED" |