diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/coccinelle/apply.sh | 9 | ||||
| -rwxr-xr-x | scripts/coccinelle/check_cocci_parse.sh | 53 | ||||
| -rw-r--r-- | scripts/coccinelle/exceptions.txt | 24 | ||||
| -rwxr-xr-x | scripts/coccinelle/test-operator-cleanup | 13 | ||||
| -rw-r--r-- | scripts/coccinelle/tor-coccinelle.h | 60 | ||||
| -rwxr-xr-x | scripts/coccinelle/try_parse.sh | 46 | ||||
| -rwxr-xr-x | scripts/git/git-merge-forward.sh | 16 | ||||
| -rwxr-xr-x | scripts/git/git-pull-all.sh | 7 | ||||
| -rwxr-xr-x | scripts/git/git-push-all.sh | 86 | ||||
| -rwxr-xr-x | scripts/git/git-setup-dirs.sh | 557 | ||||
| -rwxr-xr-x | scripts/git/pre-commit.git-hook | 78 | ||||
| -rwxr-xr-x | scripts/git/pre-push.git-hook | 159 | ||||
| -rwxr-xr-x | scripts/maint/add_c_file.py | 4 | ||||
| -rw-r--r-- | scripts/maint/practracker/.enable_practracker_in_hooks | 1 | ||||
| -rw-r--r-- | scripts/maint/practracker/exceptions.txt | 103 | ||||
| -rwxr-xr-x | scripts/maint/practracker/practracker.py | 2 | ||||
| -rw-r--r-- | scripts/maint/practracker/util.py | 6 | ||||
| -rwxr-xr-x | scripts/maint/rename_c_identifier.py | 261 | ||||
| -rwxr-xr-x | scripts/maint/update_versions.py | 2 |
19 files changed, 1319 insertions, 168 deletions
diff --git a/scripts/coccinelle/apply.sh b/scripts/coccinelle/apply.sh new file mode 100755 index 0000000000..f531d7fa32 --- /dev/null +++ b/scripts/coccinelle/apply.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +# apply.sh: +# run spatch with appropriate includes and builtins for the Tor source code + +top="$(dirname "$0")/../.." + +spatch -macro_file_builtins "$top"/scripts/coccinelle/tor-coccinelle.h \ + -I "$top" -I "$top"/src -I "$top"/ext --defined COCCI "$@" diff --git a/scripts/coccinelle/check_cocci_parse.sh b/scripts/coccinelle/check_cocci_parse.sh new file mode 100755 index 0000000000..220b405940 --- /dev/null +++ b/scripts/coccinelle/check_cocci_parse.sh @@ -0,0 +1,53 @@ +#!/bin/sh + +# If we have coccinelle installed, run try_parse.sh on every filename passed +# as an argument. If no filenames are supplied, scan a standard Tor 0.3.5 or +# later directory layout. +# +# Uses the default coccinelle exceptions file, or $TOR_COCCI_EXCEPTIONS_FILE, +# if it is set. +# +# Use TOR_COCCI_EXCEPTIONS_FILE=/dev/null check_cocci_parse.sh to disable +# the default exception file. +# +# If spatch is not installed, remind the user to install it, but exit with +# a success error status. + +scripts_cocci="$(dirname "$0")" +top="$scripts_cocci/../.." +try_parse="$scripts_cocci/try_parse.sh" + +exitcode=0 + +export TOR_COCCI_EXCEPTIONS_FILE="${TOR_COCCI_EXCEPTIONS_FILE:-$scripts_cocci/exceptions.txt}" + +if ! command -v spatch; then + echo "Install coccinelle's spatch to check cocci C parsing!" + exit "$exitcode" +fi + +if test $# -ge 1 ; then + "$try_parse" "$@" + exitcode=$? +else + cd "$top" || exit 1 + # This is the layout in 0.3.5 + "$try_parse" \ + src/lib/*/*.[ch] \ + src/core/*/*.[ch] \ + src/feature/*/*.[ch] \ + src/app/*/*.[ch] \ + src/test/*.[ch] \ + src/test/*/*.[ch] \ + src/tools/*.[ch] + exitcode=$? +fi + +if test "$exitcode" != 0 ; then + echo "Please fix these cocci parsing errors in the above files" + echo "Set VERBOSE=1 for more details" + echo "Try running test-operator-cleanup or 'make autostyle-operators'" + echo "As a last resort, you can modify scripts/coccinelle/exceptions.txt" +fi + +exit "$exitcode" diff --git a/scripts/coccinelle/exceptions.txt b/scripts/coccinelle/exceptions.txt new file mode 100644 index 0000000000..473f4b22c5 --- /dev/null +++ b/scripts/coccinelle/exceptions.txt @@ -0,0 +1,24 @@ +# A list of exception patterns for check_cocci_parse.sh +# Passed to 'grep -f' +src/lib/cc/compat_compiler.h +src/lib/container/handles.h +src/lib/container/map.c +src/lib/container/map.h +src/lib/container/order.c +src/lib/crypt_ops/crypto_rand.c +src/lib/fs/files.h +src/lib/log/util_bug.c +src/lib/pubsub/pubsub_macros.h +src/lib/smartlist_core/smartlist_foreach.h +src/lib/testsupport/testsupport.h +src/lib/tls/tortls.h +src/lib/tls/tortls_openssl.c +src/lib/tls/x509.h +src/lib/version/version.c +src/core/mainloop/connection.c +src/core/or/reasons.c +src/feature/dirclient/dirclient.c +src/feature/nodelist/networkstatus.c +src/test/test_address.c +src/test/test_hs_cache.c +src/test/test_hs_descriptor.c diff --git a/scripts/coccinelle/test-operator-cleanup b/scripts/coccinelle/test-operator-cleanup index e7822542a4..28b4d4f588 100755 --- a/scripts/coccinelle/test-operator-cleanup +++ b/scripts/coccinelle/test-operator-cleanup @@ -1,4 +1,17 @@ #!/usr/bin/perl -w -p -i +# +# Copyright (c) 2001 Matej Pfajfar. +# Copyright (c) 2001-2004, Roger Dingledine. +# Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. +# Copyright (c) 2007-2019, The Tor Project, Inc. +# See LICENSE for licensing information + +# This script looks for instances of C comparison operators as macro arguments, +# and replaces them with our OP_* equivalents. +# +# Some macros that take operators are our tt_int_op() testing macro, and the +# standard timercmp() macro. Coccinelle can't handle their syntax, however, +# unless we give them their operators as a macro too. next if m#^ */\*# or m#^ *\* #; diff --git a/scripts/coccinelle/tor-coccinelle.h b/scripts/coccinelle/tor-coccinelle.h index 8f625dcee4..44d79325eb 100644 --- a/scripts/coccinelle/tor-coccinelle.h +++ b/scripts/coccinelle/tor-coccinelle.h @@ -1,3 +1,63 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/* + * This file looks like a C header, but its purpose is a bit different. + * + * We never include it from our real C files; we only tell Coccinelle + * about it in apply.sh. + * + * It tells the Coccinelle semantic patching tool how to understand + * things that would otherwise not be good C syntax, or which would + * otherwise not make sense to it as C. It doesn't need to produce + * semantically equivalent C, or even correct C: it only has to produce + * syntactically valid C. + */ + +#define MOCK_DECL(a, b, c) a b c #define MOCK_IMPL(a, b, c) a b c #define CHECK_PRINTF(a, b) +#define CHECK_SCANF(a, b) #define STATIC static +#define EXTERN(a,b) extern a b; + +#define STMT_BEGIN do { +#define STMT_END } while (0) + +#define BUG(x) (x) +#define IF_BUG_ONCE(x) if (x) + +#define ATTR_NORETURN +#define ATTR_UNUSED +#define ATTR_CONST +#define ATTR_MALLOC +#define ATTR_WUR +#define DISABLE_GCC_WARNING(x) +#define ENABLE_GCC_WARNING(x) + +#define HANDLE_DECL(a,b,c) +#define HANDLE_IMPL(a,b,c) +#define HT_ENTRY(x) void * +#define HT_HEAD(a,b) struct ht_head +#define HT_INITIALIZER() { } +#define X509 struct x509_st +#define STACK_OF(x) struct foo_stack_t +#define TOR_TAILQ_HEAD(a,b) struct tailq_head +#define TOR_TAILQ_ENTRY(a) struct tailq_entry +#define TOR_SIMPLEQ_HEAD(a,b) struct simpleq_entry +#define TOR_SIMPLEQ_ENTRY(a) struct simpleq_entry +#define TOR_LIST_HEAD(a,b) struct list_head +#define TOR_LIST_ENTRY(a) struct list_entry +#define TOR_SLIST_HEAD(a,b) struct slist_head +#define TOR_SLIST_ENTRY(a) struct slist_entry + +#define NS_DECL(a, b, c) a b c +#define NS(a) a + +#define CONF_TEST_MEMBERS(a,b,c) +#define DUMMY_CONF_TEST_MEMBERS + +#define EAT_SEMICOLON extern int dummy__; diff --git a/scripts/coccinelle/try_parse.sh b/scripts/coccinelle/try_parse.sh new file mode 100755 index 0000000000..a90e51b4aa --- /dev/null +++ b/scripts/coccinelle/try_parse.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +# Echo the name of every argument of this script that is not "perfect" +# according to coccinelle's --parse-c. +# +# If $TOR_COCCI_EXCEPTIONS_FILE is non-empty, skip any files that match the +# patterns in the exception file, according to "grep -f" +# +# If VERBOSE is non-empty, log spatch errors and skipped files. + +top="$(dirname "$0")/../.." + +exitcode=0 + +for fn in "$@"; do + + if test "${TOR_COCCI_EXCEPTIONS_FILE}" ; then + skip_fn=$(echo "$fn" | grep -f "${TOR_COCCI_EXCEPTIONS_FILE}") + if test "${skip_fn}" ; then + if test "${VERBOSE}" != ""; then + echo "Skipping '${skip_fn}'" + fi + continue + fi + fi + + if spatch --macro-file-builtins \ + "$top"/scripts/coccinelle/tor-coccinelle.h \ + --defined COCCI \ + --parse-c "$fn" \ + 2>/dev/null | grep "perfect = 1" > /dev/null; then + : # it's perfect + else + echo "$fn" + if test "${VERBOSE}" != ""; then + spatch --macro-file-builtins \ + "$top"/scripts/coccinelle/tor-coccinelle.h \ + --defined COCCI \ + --parse-c "$fn" + fi + exitcode=1 + fi + +done + +exit "$exitcode" diff --git a/scripts/git/git-merge-forward.sh b/scripts/git/git-merge-forward.sh index bdd0da5b75..a6df479590 100755 --- a/scripts/git/git-merge-forward.sh +++ b/scripts/git/git-merge-forward.sh @@ -98,13 +98,16 @@ MAINT_040=( "maint-0.4.0" "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.0" \ "_040" "_035") MAINT_041=( "maint-0.4.1" "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.1" \ "_041" "_040") -MAINT_MASTER=( "master" "maint-0.4.1" "$GIT_PATH/$TOR_MASTER_NAME" \ - "_master" "_041") +MAINT_042=( "maint-0.4.2" "maint-0.4.1" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.2" \ + "_042" "_041") +MAINT_MASTER=( "master" "maint-0.4.2" "$GIT_PATH/$TOR_MASTER_NAME" \ + "_master" "_042") RELEASE_029=( "release-0.2.9" "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/release-0.2.9" ) RELEASE_035=( "release-0.3.5" "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.5" ) RELEASE_040=( "release-0.4.0" "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.0" ) RELEASE_041=( "release-0.4.1" "maint-0.4.1" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.1" ) +RELEASE_042=( "release-0.4.2" "maint-0.4.2" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.2" ) # The master branch path has to be the main repository thus contains the # origin that will be used to fetch the updates. All the worktrees are created @@ -117,11 +120,13 @@ ${MAINT_029_TB[0]} ${MAINT_035[0]} ${MAINT_040[0]} ${MAINT_041[0]} +${MAINT_042[0]} ${MAINT_MASTER[0]} ${RELEASE_029[0]} ${RELEASE_035[0]} ${RELEASE_040[0]} ${RELEASE_041[0]} +${RELEASE_042[0]} EOF ####################### @@ -186,6 +191,9 @@ if [ -z "$TEST_BRANCH_PREFIX" ]; then MAINT_041[@] RELEASE_041[@] + MAINT_042[@] + RELEASE_042[@] + MAINT_MASTER[@] ) @@ -201,6 +209,8 @@ else MAINT_041[@] + MAINT_042[@] + MAINT_MASTER[@] ) @@ -323,7 +333,7 @@ function merge_branch fi } -# Pull the given branch name. +# Merge origin/(branch name) into the current branch. function merge_branch_origin { local cmd="git merge --ff-only 'origin/$1'" diff --git a/scripts/git/git-pull-all.sh b/scripts/git/git-pull-all.sh index dc16066388..9e12b565b9 100755 --- a/scripts/git/git-pull-all.sh +++ b/scripts/git/git-pull-all.sh @@ -62,12 +62,14 @@ MAINT_029=( "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/maint-0.2.9" ) MAINT_035=( "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.5" ) MAINT_040=( "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.0" ) MAINT_041=( "maint-0.4.1" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.1" ) +MAINT_042=( "maint-0.4.2" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.2" ) MAINT_MASTER=( "master" "$GIT_PATH/$TOR_MASTER_NAME" ) RELEASE_029=( "release-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/release-0.2.9" ) RELEASE_035=( "release-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.5" ) RELEASE_040=( "release-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.0" ) RELEASE_041=( "release-0.4.1" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.1" ) +RELEASE_042=( "release-0.4.2" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.2" ) # The master branch path has to be the main repository thus contains the # origin that will be used to fetch the updates. All the worktrees are created @@ -80,11 +82,13 @@ ${MAINT_029[0]} ${MAINT_035[0]} ${MAINT_040[0]} ${MAINT_041[0]} +${MAINT_042[0]} ${MAINT_MASTER[0]} ${RELEASE_029[0]} ${RELEASE_035[0]} ${RELEASE_040[0]} ${RELEASE_041[0]} +${RELEASE_042[0]} EOF ########################### @@ -106,6 +110,9 @@ WORKTREE=( MAINT_041[@] RELEASE_041[@] + MAINT_042[@] + RELEASE_042[@] + MAINT_MASTER[@] ) COUNT=${#WORKTREE[@]} diff --git a/scripts/git/git-push-all.sh b/scripts/git/git-push-all.sh index 7c43fe24d8..293df03423 100755 --- a/scripts/git/git-push-all.sh +++ b/scripts/git/git-push-all.sh @@ -29,6 +29,8 @@ function usage() echo " CI environment failures, using code that previously passed CI." echo " (default: skip; current: $CURRENT_PUSH_SAME matching branches)" echo " --: pass further arguments to git push." + echo " All unrecognised arguments are passed to git push, but complex" + echo " arguments before -- may be mangled by getopt." echo " (default: git push --atomic, current: $GIT_PUSH)" echo echo " env vars:" @@ -127,9 +129,11 @@ while getopts ":hr:st:" opt; do OPTIND=$((OPTIND - 2)) ;; *) - # Assume we're done with script arguments, - # and git push will handle the option - break + # Make git push handle the option + # This might mangle options with spaces, use -- for complex options + GIT_PUSH="$GIT_PUSH $1" + shift + OPTIND=$((OPTIND - 1)) ;; esac done @@ -151,7 +155,7 @@ if [ "$TEST_BRANCH_PREFIX" ]; then fi if [ "$TOR_GIT_PUSH_PATH" ]; then - echo "Changing to $GIT_PUSH_PATH before pushing" + echo "Changing to $TOR_GIT_PUSH_PATH before pushing" cd "$TOR_GIT_PUSH_PATH" else echo "Pushing from the current directory" @@ -167,6 +171,7 @@ DEFAULT_UPSTREAM_BRANCHES= if [ "$DEFAULT_UPSTREAM_REMOTE" != "$UPSTREAM_REMOTE" ]; then DEFAULT_UPSTREAM_BRANCHES=$(echo \ "$DEFAULT_UPSTREAM_REMOTE"/master \ + "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.4.2 \ "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.4.1 \ "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.4.0 \ "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.3.5 \ @@ -176,6 +181,7 @@ fi UPSTREAM_BRANCHES=$(echo \ "$UPSTREAM_REMOTE"/master \ + "$UPSTREAM_REMOTE"/{release,maint}-0.4.2 \ "$UPSTREAM_REMOTE"/{release,maint}-0.4.1 \ "$UPSTREAM_REMOTE"/{release,maint}-0.4.0 \ "$UPSTREAM_REMOTE"/{release,maint}-0.3.5 \ @@ -188,6 +194,7 @@ UPSTREAM_BRANCHES=$(echo \ PUSH_BRANCHES=$(echo \ master \ + {release,maint}-0.4.2 \ {release,maint}-0.4.1 \ {release,maint}-0.4.0 \ {release,maint}-0.3.5 \ @@ -201,6 +208,7 @@ if [ -z "$TEST_BRANCH_PREFIX" ]; then # List of branches to push. Ordering is not important. PUSH_BRANCHES=$(echo \ master \ + {release,maint}-0.4.2 \ {release,maint}-0.4.1 \ {release,maint}-0.4.0 \ {release,maint}-0.3.5 \ @@ -213,6 +221,7 @@ else # List of branches to push. Ordering is not important. PUSH_BRANCHES=" \ ${TEST_BRANCH_PREFIX}_master \ + ${TEST_BRANCH_PREFIX}_042 \ ${TEST_BRANCH_PREFIX}_041 \ ${TEST_BRANCH_PREFIX}_040 \ ${TEST_BRANCH_PREFIX}_035 \ @@ -224,20 +233,32 @@ fi # Entry point # ############### -# Skip the test branches that are the same as the upstream branches -if [ "$PUSH_SAME" -eq 0 ] && [ "$TEST_BRANCH_PREFIX" ]; then +if [ "$TEST_BRANCH_PREFIX" ]; then + # Skip the test branches that are the same as the default or current + # upstream branches (they have already been tested) + UPSTREAM_SKIP_SAME_AS="$UPSTREAM_BRANCHES $DEFAULT_UPSTREAM_BRANCHES" +else + # Skip the local maint-*, release-*, master branches that are the same as the + # current upstream branches, but ignore the default upstream + # (we want to update a non-default remote, even if it matches the default) + UPSTREAM_SKIP_SAME_AS="$UPSTREAM_BRANCHES" +fi + +# Skip branches that match the relevant upstream(s) +if [ "$PUSH_SAME" -eq 0 ]; then NEW_PUSH_BRANCHES= for b in $PUSH_BRANCHES; do PUSH_COMMIT=$(git rev-parse "$b") SKIP_UPSTREAM= - for u in $DEFAULT_UPSTREAM_BRANCHES $UPSTREAM_BRANCHES; do - UPSTREAM_COMMIT=$(git rev-parse "$u") + for u in $UPSTREAM_SKIP_SAME_AS; do + # Skip the branch check on error + UPSTREAM_COMMIT=$(git rev-parse "$u" 2>/dev/null) || continue if [ "$PUSH_COMMIT" = "$UPSTREAM_COMMIT" ]; then SKIP_UPSTREAM="$u" fi done if [ "$SKIP_UPSTREAM" ]; then - printf "Skipping unchanged: %s remote: %s\\n" \ + printf "Skipping unchanged: %s matching remote: %s\\n" \ "$b" "$SKIP_UPSTREAM" else if [ "$NEW_PUSH_BRANCHES" ]; then @@ -250,6 +271,12 @@ if [ "$PUSH_SAME" -eq 0 ] && [ "$TEST_BRANCH_PREFIX" ]; then PUSH_BRANCHES=${NEW_PUSH_BRANCHES} fi +if [ ! "$PUSH_BRANCHES" ]; then + echo "No branches to push!" + # We expect the rest of the script to run without errors, even if there + # are no branches +fi + if [ "$PUSH_DELAY" -le 0 ]; then echo "Pushing $PUSH_BRANCHES" # We know that there are no spaces in any branch within $PUSH_BRANCHES, so @@ -262,28 +289,43 @@ if [ "$PUSH_DELAY" -le 0 ]; then else # Push the branches in optimal CI order, with a delay between each push PUSH_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | sort -V) - MASTER_BRANCH=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep master) + MASTER_BRANCH=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep master) \ + || true # Skipped master branch if [ -z "$TEST_BRANCH_PREFIX" ]; then - MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep maint) + MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep maint) \ + || true # Skipped all maint branches RELEASE_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep release | \ - tr "\\n" " ") - printf \ - "Pushing with %ss delays, so CI runs in this order:\\n%s\\n%s\\n%s\\n" \ - "$PUSH_DELAY" "$MASTER_BRANCH" "$MAINT_BRANCHES" "$RELEASE_BRANCHES" + tr "\\n" " ") || true # Skipped all release branches else # Actually test branches based on maint branches - MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep -v master) - printf "Pushing with %ss delays, so CI runs in this order:\\n%s\\n%s\\n" \ - "$PUSH_DELAY" "$MASTER_BRANCH" "$MAINT_BRANCHES" + MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep -v master) \ + || true # Skipped all maint test branches # No release branches RELEASE_BRANCHES= fi - $GIT_PUSH "$@" "$UPSTREAM_REMOTE" "$MASTER_BRANCH" - sleep "$PUSH_DELAY" + if [ "$MASTER_BRANCH" ] || [ "$MAINT_BRANCHES" ] \ + || [ "$RELEASE_BRANCHES" ]; then + printf "Pushing with %ss delays, so CI runs in this order:\\n" \ + "$PUSH_DELAY" + if [ "$MASTER_BRANCH" ]; then + printf "%s\\n" "$MASTER_BRANCH" + fi + if [ "$MAINT_BRANCHES" ]; then + printf "%s\\n" "$MAINT_BRANCHES" + fi + if [ "$RELEASE_BRANCHES" ]; then + printf "%s\\n" "$RELEASE_BRANCHES" + fi + fi # shellcheck disable=SC2086 - for b in $MAINT_BRANCHES; do + for b in $MASTER_BRANCH $MAINT_BRANCHES; do $GIT_PUSH "$@" "$UPSTREAM_REMOTE" "$b" - sleep "$PUSH_DELAY" + # If we are pushing more than one branch, delay. + # In the unlikely scenario where we are pushing maint without master, + # or maint without release, there may be an extra delay + if [ "$MAINT_BRANCHES" ] || [ "$RELEASE_BRANCHES" ]; then + sleep "$PUSH_DELAY" + fi done if [ "$RELEASE_BRANCHES" ]; then # shellcheck disable=SC2086 diff --git a/scripts/git/git-setup-dirs.sh b/scripts/git/git-setup-dirs.sh new file mode 100755 index 0000000000..b7a37a04eb --- /dev/null +++ b/scripts/git/git-setup-dirs.sh @@ -0,0 +1,557 @@ +#!/usr/bin/env bash + +SCRIPT_NAME=$(basename "$0") + +function usage() +{ + echo "$SCRIPT_NAME [-h] [-n] [-u]" + echo + echo " arguments:" + echo " -h: show this help text" + echo " -n: dry run mode" + echo " (default: run commands)" + echo " -u: if a directory or worktree already exists, use it" + echo " (default: fail and exit on existing directories)" + echo + echo " env vars:" + echo " required:" + echo " TOR_FULL_GIT_PATH: where the git repository directories reside." + echo " You must set this env var, we recommend \$HOME/git/" + echo " (default: fail if this env var is not set;" + echo " current: $GIT_PATH)" + echo + echo " optional:" + echo " TOR_MASTER: the name of the directory containing the tor.git clone" + echo " The tor master git directory is \$GIT_PATH/\$TOR_MASTER" + echo " (default: tor; current: $TOR_MASTER_NAME)" + echo " TOR_WKT_NAME: the name of the directory containing the tor" + echo " worktrees. The tor worktrees are:" + echo " \$GIT_PATH/\$TOR_WKT_NAME/{maint-*,release-*}" + echo " (default: tor-wkt; current: $TOR_WKT_NAME)" + echo " TOR_GIT_ORIGIN_PULL: the origin remote pull URL." + echo " (current: $GIT_ORIGIN_PULL)" + echo " TOR_GIT_ORIGIN_PUSH: the origin remote push URL" + echo " (current: $GIT_ORIGIN_PUSH)" + echo " TOR_UPSTREAM_REMOTE_NAME: the default upstream remote." + echo " If \$TOR_UPSTREAM_REMOTE_NAME is not 'origin', we have a" + echo " separate upstream remote, and we don't push to origin." + echo " (default: $DEFAULT_UPSTREAM_REMOTE)" + echo " TOR_GITHUB_PULL: the tor-github remote pull URL" + echo " (current: $GITHUB_PULL)" + echo " TOR_GITHUB_PUSH: the tor-github remote push URL" + echo " (current: $GITHUB_PUSH)" + echo " TOR_EXTRA_CLONE_ARGS: extra arguments to git clone" + echo " (current: $TOR_EXTRA_CLONE_ARGS)" + echo " TOR_EXTRA_REMOTE_NAME: the name of an extra remote" + echo " This remote is not pulled by this script or git-pull-all.sh." + echo " This remote is not pushed by git-push-all.sh." + echo " (current: $TOR_EXTRA_REMOTE_NAME)" + echo " TOR_EXTRA_REMOTE_PULL: the extra remote pull URL." + echo " (current: $TOR_EXTRA_REMOTE_PULL)" + echo " TOR_EXTRA_REMOTE_PUSH: the extra remote push URL" + echo " (current: $TOR_EXTRA_REMOTE_PUSH)" + echo " we recommend that you set these env vars in your ~/.profile" +} + +################# +# Configuration # +################# + +# Don't change this configuration - set the env vars in your .profile + +# Where are all those git repositories? +GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"} +# The tor master git repository directory from which all the worktree have +# been created. +TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"} +# The worktrees location (directory). +TOR_WKT_NAME=${TOR_WKT_NAME:-"tor-wkt"} + +# Origin repositories +GIT_ORIGIN_PULL=${TOR_GIT_ORIGIN_PULL:-"https://git.torproject.org/tor.git"} +GIT_ORIGIN_PUSH=${TOR_GIT_ORIGIN_PUSH:-"git@git-rw.torproject.org:tor.git"} +# The upstream remote which git.torproject.org/tor.git points to. +DEFAULT_UPSTREAM_REMOTE=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"} +# Copy the URLs from origin +GIT_UPSTREAM_PULL="$GIT_ORIGIN_PULL" +GIT_UPSTREAM_PUSH="$GIT_ORIGIN_PUSH" +# And avoid pushing to origin if we have an upstream +if [ "$DEFAULT_UPSTREAM_REMOTE" != "origin" ]; then + GIT_ORIGIN_PUSH="No pushes to origin, if there is an upstream" +fi +# GitHub repositories +GITHUB_PULL=${TOR_GITHUB_PULL:-"https://github.com/torproject/tor.git"} +GITHUB_PUSH=${TOR_GITHUB_PUSH:-"No_Pushing_To_GitHub"} + +########################## +# Git branches to manage # +########################## + +# The branches and worktrees need to be modified when there is a new branch, +# and when an old branch is no longer supported. + +# Configuration of the branches that needs merging. The values are in order: +# (0) current maint/release branch name +# (1) Full path of the git worktree +# +# First set of arrays are the maint-* branch and then the release-* branch. +# New arrays need to be in the WORKTREE= array else they aren't considered. +MAINT_029=( "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/maint-0.2.9" ) +MAINT_035=( "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.5" ) +MAINT_040=( "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.0" ) +MAINT_041=( "maint-0.4.1" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.1" ) +MAINT_042=( "maint-0.4.2" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.2" ) +MAINT_MASTER=( "master" "$GIT_PATH/$TOR_MASTER_NAME" ) + +RELEASE_029=( "release-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/release-0.2.9" ) +RELEASE_035=( "release-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.5" ) +RELEASE_040=( "release-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.0" ) +RELEASE_041=( "release-0.4.1" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.1" ) +RELEASE_042=( "release-0.4.2" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.2" ) + +# The master branch path has to be the main repository thus contains the +# origin that will be used to fetch the updates. All the worktrees are created +# from that repository. +ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME" + +# SC2034 -- shellcheck thinks that these are unused. We know better. +ACTUALLY_THESE_ARE_USED=<<EOF +${MAINT_029[0]} +${MAINT_035[0]} +${MAINT_040[0]} +${MAINT_041[0]} +${MAINT_042[0]} +${MAINT_MASTER[0]} +${RELEASE_029[0]} +${RELEASE_035[0]} +${RELEASE_040[0]} +${RELEASE_041[0]} +${RELEASE_042[0]} +EOF + +####################### +# Argument processing # +####################### + +# Controlled by the -n option. The dry run option will just output the command +# that would have been executed for each worktree. +DRY_RUN=0 + +# Controlled by the -s option. The use existing option checks for existing +# directories, and re-uses them, rather than creating a new directory. +USE_EXISTING=0 +USE_EXISTING_HINT="Use existing: '$SCRIPT_NAME -u'." + +while getopts "hnu" opt; do + case "$opt" in + h) usage + exit 0 + ;; + n) DRY_RUN=1 + echo " *** DRY RUN MODE ***" + ;; + u) USE_EXISTING=1 + echo " *** USE EXISTING DIRECTORIES MODE ***" + ;; + *) + echo + usage + exit 1 + ;; + esac +done + +########################### +# Git worktrees to manage # +########################### + +WORKTREE=( + MAINT_029[@] + RELEASE_029[@] + + MAINT_035[@] + RELEASE_035[@] + + MAINT_040[@] + RELEASE_040[@] + + MAINT_041[@] + RELEASE_041[@] + + MAINT_042[@] + RELEASE_042[@] + + MAINT_MASTER[@] +) + +COUNT=${#WORKTREE[@]} + +############# +# Constants # +############# + +# Control characters +CNRM=$'\x1b[0;0m' # Clear color + +# Bright color +BGRN=$'\x1b[1;32m' +BBLU=$'\x1b[1;34m' +BRED=$'\x1b[1;31m' +BYEL=$'\x1b[1;33m' +IWTH=$'\x1b[3;37m' + +# Strings for the pretty print. +MARKER="${BBLU}[${BGRN}+${BBLU}]${CNRM}" +SUCCESS="${BGRN}success${CNRM}" +SKIPPED="${BYEL}skipped${CNRM}" +FAILED="${BRED}failed${CNRM}" + +#################### +# Helper functions # +#################### + +# Validate the given returned value (error code), print success or failed. The +# second argument is the error output in case of failure, it is printed out. +# On failure, this function exits. +function validate_ret +{ + if [ "$1" -eq 0 ]; then + printf "%s\\n" "$SUCCESS" + else + printf "%s\\n" "$FAILED" + printf " %s\\n" "$2" + exit 1 + fi +} + +# Validate the given returned value (error code), print success, skipped, or +# failed. If $USE_EXISTING is 0, fail on error, otherwise, skip on error. +# The second argument is the error output in case of failure, it is printed +# out. On failure, this function exits. +function validate_ret_skip +{ + if [ "$1" -ne 0 ]; then + if [ "$USE_EXISTING" -eq "0" ]; then + # Fail and exit with error + validate_ret "$1" "$2 $USE_EXISTING_HINT" + else + printf "%s\\n" "$SKIPPED" + printf " %s\\n" "${IWTH}$2${CNRM}" + # Tell the caller to skip the rest of the function + return 0 + fi + fi + # Tell the caller to continue + return 1 +} + +# Create a directory, and any missing enclosing directories. +# If the directory already exists: fail if $USE_EXISTING is 0, otherwise skip. +function make_directory +{ + local cmd="mkdir -p '$1'" + printf " %s Creating directory %s..." "$MARKER" "$1" + local check_cmd="[ ! -d '$1' ]" + msg=$( eval "$check_cmd" 2>&1 ) + if validate_ret_skip $? "Directory already exists."; then + return + fi + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Create a symlink from the first argument to the second argument +# If the link already exists: fail if $USE_EXISTING is 0, otherwise skip. +function make_symlink +{ + local cmd="ln -s '$1' '$2'" + printf " %s Creating symlink from %s to %s..." "$MARKER" "$1" "$2" + local check_cmd="[ ! -e '$2' ]" + msg=$( eval "$check_cmd" 2>&1 ) + if validate_ret_skip $? "File already exists."; then + return + fi + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Go into the directory or repository, even if $DRY_RUN is non-zero. +# If the directory does not exist, fail and log an error. +# Otherwise, silently succeed. +function goto_dir +{ + if ! cd "$1" 1>/dev/null 2>/dev/null ; then + printf " %s Changing to directory %s..." "$MARKER" "$1" + validate_ret 1 "$1: Not found. Stopping." + fi +} + +# Clone a repository into a directory. +# If the directory already exists: fail if $USE_EXISTING is 0, otherwise skip. +function clone_repo +{ + local cmd="git clone $TOR_EXTRA_CLONE_ARGS '$1' '$2'" + printf " %s Cloning %s into %s..." "$MARKER" "$1" "$2" + local check_cmd="[ ! -d '$2' ]" + msg=$( eval "$check_cmd" 2>&1 ) + if validate_ret_skip $? "Directory already exists."; then + # If we skip the clone, we need to do a fetch + goto_dir "$ORIGIN_PATH" + fetch_remote "origin" + return + fi + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Add a remote by name and URL. +# If the remote already exists: fail if $USE_EXISTING is 0, otherwise skip. +function add_remote +{ + local cmd="git remote add '$1' '$2'" + printf " %s Adding remote %s at %s..." "$MARKER" "$1" "$2" + local check_cmd="git remote get-url '$1'" + msg=$( eval "$check_cmd" 2>&1 ) + ret=$? + # We don't want a remote, so we invert the exit status + if validate_ret_skip $(( ! ret )) \ + "Remote already exists for $1 at $msg."; then + return + fi + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Set a remote's push URL by name and URL. +function set_remote_push +{ + local cmd="git remote set-url --push '$1' '$2'" + printf " %s Setting remote %s push URL to '%s'..." "$MARKER" "$1" "$2" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Fetch a remote by name. +function fetch_remote +{ + local cmd="git fetch '$1'" + printf " %s Fetching %s..." "$MARKER" "$1" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Replace the fetch configs for a remote with config if they match a pattern. +function replace_fetch_config +{ + local cmd="git config --replace-all remote.'$1'.fetch '$2' '$3'" + printf " %s Replacing %s fetch configs for '%s'..." \ + "$MARKER" "$1" "$3" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Set up the tor-github PR config, so tor-github/pr/NNNN/head points to GitHub +# PR NNNN. In some repositories, "/head" is optional. +function set_tor_github_pr_fetch_config +{ + # Standard branches + replace_fetch_config tor-github \ + "+refs/heads/*:refs/remotes/tor-github/*" \ + "refs/heads" + # PRs + replace_fetch_config "tor-github" \ + "+refs/pull/*:refs/remotes/tor-github/pr/*" \ + "refs/pull.*pr" +} + +# Add a new worktree for branch at path. +# If the directory already exists: fail if $USE_EXISTING is 0, otherwise skip. +function add_worktree +{ + local cmd="git worktree add '$2' '$1'" + printf " %s Adding worktree for %s at %s..." "$MARKER" "$1" "$2" + local check_cmd="[ ! -d '$2' ]" + msg=$( eval "$check_cmd" 2>&1 ) + if validate_ret_skip $? "Directory already exists."; then + return + fi + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Switch to the given branch name. +# If the branch does not exist: fail. +function switch_branch +{ + local cmd="git checkout '$1'" + printf " %s Switching branch to %s..." "$MARKER" "$1" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Checkout a new branch with the given branch name. +# If the branch already exists: fail if $USE_EXISTING is 0, otherwise skip. +function new_branch +{ + local cmd="git checkout -b '$1'" + printf " %s Creating new branch %s..." "$MARKER" "$1" + local check_cmd="git branch --list '$1'" + msg=$( eval "$check_cmd" 2>&1 ) + if validate_ret_skip $? "Branch already exists."; then + return + fi + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Switch to an existing branch, or checkout a new branch with the given +# branch name. +function switch_or_new_branch +{ + local cmd="git rev-parse --verify '$1'" + if [ $DRY_RUN -eq 0 ]; then + # Call switch_branch if there is a branch, or new_branch if there is not + msg=$( eval "$cmd" 2>&1 ) + RET=$? + if [ $RET -eq 0 ]; then + # Branch: (commit id) + switch_branch "$1" + elif [ $RET -eq 128 ]; then + # Not a branch: "fatal: Needed a single revision" + new_branch "$1" + else + # Unexpected return value + validate_ret $RET "$msg" + fi + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}, then depending on the result:" + switch_branch "$1" + new_branch "$1" + fi +} + +# Set the upstream for branch to upstream. +function set_upstream +{ + # Note the argument order is swapped + local cmd="git branch --set-upstream-to='$2' '$1'" + printf " %s Setting upstream for %s to %s..." "$MARKER" "$1" "$2" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +############### +# Entry point # +############### + +printf "%s Setting up the repository and remote %s\\n" "$MARKER" \ + "${BYEL}origin${CNRM}" +# First, fetch the origin. +ORIGIN_PARENT=$(dirname "$ORIGIN_PATH") +make_directory "$ORIGIN_PARENT" +# This is just cd with an error check +goto_dir "$ORIGIN_PARENT" + +# clone repository / origin remote +clone_repo "$GIT_ORIGIN_PULL" "$TOR_MASTER_NAME" +goto_dir "$ORIGIN_PATH" +set_remote_push "origin" "$GIT_ORIGIN_PUSH" + +# upstream remote, if different to origin +if [ "$DEFAULT_UPSTREAM_REMOTE" != "origin" ]; then + printf "%s Setting up remote %s\\n" "$MARKER" \ + "${BYEL}$DEFAULT_UPSTREAM_REMOTE${CNRM}" + add_remote "$DEFAULT_UPSTREAM_REMOTE" "$GIT_UPSTREAM_PULL" + set_remote_push "$DEFAULT_UPSTREAM_REMOTE" "$GIT_UPSTREAM_PUSH" + fetch_remote "$DEFAULT_UPSTREAM_REMOTE" +fi + +# GitHub remote +printf "%s Setting up remote %s\\n" "$MARKER" "${BYEL}tor-github${CNRM}" +# Add remote +add_remote "tor-github" "$GITHUB_PULL" +set_remote_push "tor-github" "$GITHUB_PUSH" +# Add custom fetch for PRs +set_tor_github_pr_fetch_config +# Now fetch them all +fetch_remote "tor-github" + +# Extra remote +if [ "$TOR_EXTRA_REMOTE_NAME" ]; then + printf "%s Setting up remote %s\\n" "$MARKER" \ + "${BYEL}$TOR_EXTRA_REMOTE_NAME${CNRM}" + # Add remote + add_remote "$TOR_EXTRA_REMOTE_NAME" "$TOR_EXTRA_REMOTE_PULL" + set_remote_push "$TOR_EXTRA_REMOTE_NAME" "$TOR_EXTRA_REMOTE_PUSH" + # But leave it to the user to decide if they want to fetch it + #fetch_remote "$TOR_EXTRA_REMOTE_NAME" +fi + +# Go over all configured worktree. +for ((i=0; i<COUNT; i++)); do + branch=${!WORKTREE[$i]:0:1} + repo_path=${!WORKTREE[$i]:1:1} + + printf "%s Handling branch %s\\n" "$MARKER" "${BYEL}$branch${CNRM}" + # We cloned the repository, and master is the default branch + if [ "$branch" = "master" ]; then + if [ "$TOR_MASTER_NAME" != "master" ]; then + # Set up a master link in the worktree directory + make_symlink "$repo_path" "$GIT_PATH/$TOR_WKT_NAME/master" + fi + else + # git makes worktree directories if they don't exist + add_worktree "origin/$branch" "$repo_path" + fi + goto_dir "$repo_path" + switch_or_new_branch "$branch" + set_upstream "$branch" "origin/$branch" +done + +echo +echo "Remember to copy the git hooks from tor/scripts/git/*.git-hook to" +echo "$ORIGIN_PATH/.git/hooks/*" diff --git a/scripts/git/pre-commit.git-hook b/scripts/git/pre-commit.git-hook index 1c381ec60a..c138d2ae21 100755 --- a/scripts/git/pre-commit.git-hook +++ b/scripts/git/pre-commit.git-hook @@ -13,30 +13,61 @@ cd "$workdir" || exit 1 set -e +if [ $# -eq 0 ]; then + # When called in pre-commit, check the files modified in this commit + CHECK_FILTER="git diff --cached --name-only --diff-filter=ACMR" + # Use the appropriate owned tor source list to filter the changed files + if [ -d src/lib ]; then + # This is the layout in 0.3.5 + CHECK_FILES="$($CHECK_FILTER \ + src/lib/*/*.[ch] \ + src/core/*/*.[ch] \ + src/feature/*/*.[ch] \ + src/app/*/*.[ch] \ + src/test/*.[ch] \ + src/test/*/*.[ch] \ + src/tools/*.[ch] \ + )" + elif [ -d src/common ]; then + # This was the layout before 0.3.5 + CHECK_FILES="$($CHECK_FILTER \ + src/common/*/*.[ch] \ + src/or/*/*.[ch] \ + src/test/*.[ch] \ + src/test/*/*.[ch] \ + src/tools/*.[ch] + )" + fi +else + # When called in pre-push, concatenate the argument array + # Fails on special characters in file names + CHECK_FILES="$*" +fi + +## General File Checks + if [ -n "$(ls ./changes/)" ]; then python scripts/maint/lintChanges.py ./changes/* fi -if [ -d src/lib ]; then - # This is the layout in 0.3.5 - perl scripts/maint/checkSpace.pl -C \ - src/lib/*/*.[ch] \ - src/core/*/*.[ch] \ - src/feature/*/*.[ch] \ - src/app/*/*.[ch] \ - src/test/*.[ch] \ - src/test/*/*.[ch] \ - src/tools/*.[ch] -elif [ -d src/common ]; then - # This was the layout before 0.3.5 - perl scripts/maint/checkSpace.pl -C \ - src/common/*/*.[ch] \ - src/or/*/*.[ch] \ - src/test/*.[ch] \ - src/test/*/*.[ch] \ - src/tools/*.[ch] +if [ -e scripts/maint/checkShellScripts.sh ]; then + scripts/maint/checkShellScripts.sh +fi + +if [ ! "$CHECK_FILES" ]; then + echo "No modified tor-owned source files, skipping further checks" + exit 0 fi +## Owned Source File Checks + +printf "Modified tor-owned source files:\\n%s\\n" "$CHECK_FILES" + +# We want word splitting here, because file names are space separated +# shellcheck disable=SC2086 +perl scripts/maint/checkSpace.pl -C \ + $CHECK_FILES + if test -e scripts/maint/practracker/includes.py; then python scripts/maint/practracker/includes.py fi @@ -54,6 +85,13 @@ if [ -e "${PT_DIR}/practracker.py" ]; then fi fi -if [ -e scripts/maint/checkShellScripts.sh ]; then - scripts/maint/checkShellScripts.sh +if [ -e scripts/coccinelle/check_cocci_parse.sh ]; then + + # Run a verbose cocci parse check on the changed files + # (spatch is slow, so we don't want to check all the files.) + # + # We want word splitting here, because file names are space separated + # shellcheck disable=SC2086 + VERBOSE=1 scripts/coccinelle/check_cocci_parse.sh \ + $CHECK_FILES fi diff --git a/scripts/git/pre-push.git-hook b/scripts/git/pre-push.git-hook index f4504c4215..6a85e951a8 100755 --- a/scripts/git/pre-push.git-hook +++ b/scripts/git/pre-push.git-hook @@ -16,91 +16,116 @@ # The following sample script was used as starting point: # https://github.com/git/git/blob/master/templates/hooks--pre-push.sample +# Are you adding a new check to the git hooks? +# - Common checks belong in the pre-commit hook +# - Push-only checks belong in the pre-push hook + echo "Running pre-push hook" z40=0000000000000000000000000000000000000000 upstream_name=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"} -# Are you adding a new check to the git hooks? -# - Common checks belong in the pre-commit hook -# - Push-only checks belong in the pre-push hook -# -# Call the pre-commit hook for the common checks, if it is executable. workdir=$(git rev-parse --show-toplevel) -if [ -x "$workdir/.git/hooks/pre-commit" ]; then - if ! "$workdir"/.git/hooks/pre-commit; then - exit 1 - fi -fi -remote="$1" +cd "$workdir" || exit 1 +remote="$1" remote_name=$(git remote --verbose | grep "$2" | awk '{print $1}' | head -n 1) -if [[ "$remote_name" != "$upstream_name" ]]; then - echo "Not pushing to upstream - refraining from further checks" - exit 0 -fi ref_is_upstream_branch() { - if [ "$1" == "refs/heads/master" ] || - [[ "$1" == refs/heads/release-* ]] || - [[ "$1" == refs/heads/maint-* ]] - then - return 1 - fi + if [ "$1" == "refs/heads/master" ] || + [[ "$1" == refs/heads/release-* ]] || + [[ "$1" == refs/heads/maint-* ]]; then + return 1 + fi } # shellcheck disable=SC2034 while read -r local_ref local_sha remote_ref remote_sha do - if [ "$local_sha" = $z40 ] - then - # Handle delete - : - else - if [ "$remote_sha" = $z40 ] - then - # New branch, examine all commits - range="$local_sha" - else - # Update to existing branch, examine new commits - range="$remote_sha..$local_sha" - fi - - if (ref_is_upstream_branch "$local_ref" == 0 || - ref_is_upstream_branch "$remote_ref" == 0) && - [ "$local_ref" != "$remote_ref" ] - then - if [ "$remote" == "origin" ] - then - echo >&2 "Not pushing: $local_ref to $remote_ref" - echo >&2 "If you really want to push this, use --no-verify." - exit 1 - else - continue - fi - fi - - # Check for fixup! commit - commit=$(git rev-list -n 1 --grep '^fixup!' "$range") - if [ -n "$commit" ] - then - echo >&2 "Found fixup! commit in $local_ref, not pushing" - echo >&2 "If you really want to push this, use --no-verify." - exit 1 - fi - - # Check for squash! commit - commit=$(git rev-list -n 1 --grep '^squash!' "$range") - if [ -n "$commit" ] - then - echo >&2 "Found squash! commit in $local_ref, not pushing" - echo >&2 "If you really want to push this, use --no-verify." - exit 1 - fi - fi + if [ "$local_sha" = $z40 ]; then + # Handle delete + : + else + if [ "$remote_sha" = $z40 ]; then + # New branch, examine commits not in master + range="master...$local_sha" + else + # Update to existing branch, examine new commits + range="$remote_sha..$local_sha" + fi + + # Call the pre-commit hook for the common checks, if it is executable + if [ -x scripts/git/pre-commit.git-hook ]; then + # Only check the files newly modified in this branch + CHECK_FILTER="git diff --name-only --diff-filter=ACMR $range" + # Use the appropriate owned tor source list to filter the changed + # files + if [ -d src/lib ]; then + # This is the layout in 0.3.5 + CHECK_FILES="$($CHECK_FILTER \ + src/lib/*/*.[ch] \ + src/core/*/*.[ch] \ + src/feature/*/*.[ch] \ + src/app/*/*.[ch] \ + src/test/*.[ch] \ + src/test/*/*.[ch] \ + src/tools/*.[ch] \ + )" + elif [ -d src/common ]; then + # This was the layout before 0.3.5 + CHECK_FILES="$($CHECK_FILTER \ + src/common/*/*.[ch] \ + src/or/*/*.[ch] \ + src/test/*.[ch] \ + src/test/*/*.[ch] \ + src/tools/*.[ch] + )" + fi + + # We want word splitting here, because file names are space + # separated + # shellcheck disable=SC2086 + if ! scripts/git/pre-commit.git-hook $CHECK_FILES ; then + exit 1 + fi + fi + + if [[ "$remote_name" != "$upstream_name" ]]; then + echo "Not pushing to upstream - refraining from further checks" + continue + fi + + if (ref_is_upstream_branch "$local_ref" == 0 || + ref_is_upstream_branch "$remote_ref" == 0) && + [ "$local_ref" != "$remote_ref" ]; then + if [ "$remote" == "origin" ]; then + echo >&2 "Not pushing: $local_ref to $remote_ref" + echo >&2 "If you really want to push this, use --no-verify." + exit 1 + else + continue + fi + fi + + # Check for fixup! commit + commit=$(git rev-list -n 1 --grep '^fixup!' "$range") + if [ -n "$commit" ]; then + echo >&2 "Found fixup! commit in $local_ref, not pushing" + echo >&2 "If you really want to push this, use --no-verify." + exit 1 + fi + + # Check for squash! commit + commit=$(git rev-list -n 1 --grep '^squash!' "$range") + if [ -n "$commit" ]; then + echo >&2 "Found squash! commit in $local_ref, not pushing" + echo >&2 "If you really want to push this, use --no-verify." + exit 1 + fi + fi done exit 0 diff --git a/scripts/maint/add_c_file.py b/scripts/maint/add_c_file.py index adf7ce79bb..a773fd0fff 100755 --- a/scripts/maint/add_c_file.py +++ b/scripts/maint/add_c_file.py @@ -15,9 +15,7 @@ import time def topdir_file(name): """Strip opening "src" from a filename""" - if name.startswith("src/"): - name = name[4:] - return name + return os.path.relpath(name, './src') def guard_macro(name): """Return the guard macro that should be used for the header file 'name'. diff --git a/scripts/maint/practracker/.enable_practracker_in_hooks b/scripts/maint/practracker/.enable_practracker_in_hooks new file mode 100644 index 0000000000..a9e707f5da --- /dev/null +++ b/scripts/maint/practracker/.enable_practracker_in_hooks @@ -0,0 +1 @@ +This file is present to tell our git hooks to run practracker on this branch. diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 7b15b37f8c..1c3bf9cbeb 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -6,8 +6,12 @@ # # There are three kinds of problems that we recognize right now: # function-size -- a function of more than 100 lines. -# file-size -- a file of more than 3000 lines. -# include-count -- a file with more than 50 #includes. +# file-size -- a .c file of more than 3000 lines, or a .h +# file with more than 500 lines. +# include-count -- a .c file with more than 50 #includes, +# or a .h file with more than 15 #includes. +# dependency-violation -- a file includes a header that it should +# not, according to an advisory .may_include file. # # Each line below represents a single exception that practracker should # _ignore_. Each line has four parts: @@ -29,34 +33,35 @@ # # Remember: It is better to fix the problem than to add a new exception! -problem file-size /src/app/config/config.c 8518 -problem include-count /src/app/config/config.c 89 -problem function-size /src/app/config/config.c:options_act_reversible() 296 -problem function-size /src/app/config/config.c:options_act() 589 +problem file-size /src/app/config/config.c 7400 +problem include-count /src/app/config/config.c 80 +problem function-size /src/app/config/config.c:options_act_reversible() 298 +problem function-size /src/app/config/config.c:options_act() 381 problem function-size /src/app/config/config.c:resolve_my_address() 190 -problem function-size /src/app/config/config.c:options_validate() 1209 -problem function-size /src/app/config/config.c:options_init_from_torrc() 207 -problem function-size /src/app/config/config.c:options_init_from_string() 171 -problem function-size /src/app/config/config.c:options_init_logs() 145 +problem function-size /src/app/config/config.c:options_validate_cb() 780 +problem function-size /src/app/config/config.c:options_init_from_torrc() 188 +problem function-size /src/app/config/config.c:options_init_from_string() 103 +problem function-size /src/app/config/config.c:options_init_logs() 125 problem function-size /src/app/config/config.c:parse_bridge_line() 104 -problem function-size /src/app/config/config.c:parse_transport_line() 189 +problem function-size /src/app/config/config.c:pt_parse_transport_line() 189 problem function-size /src/app/config/config.c:parse_dir_authority_line() 150 problem function-size /src/app/config/config.c:parse_dir_fallback_line() 101 -problem function-size /src/app/config/config.c:parse_port_config() 446 -problem function-size /src/app/config/config.c:parse_ports() 168 -problem file-size /src/app/config/or_options_st.h 1112 -problem include-count /src/app/main/main.c 68 +problem function-size /src/app/config/config.c:port_parse_config() 450 +problem function-size /src/app/config/config.c:parse_ports() 132 +problem file-size /src/app/config/or_options_st.h 1115 +problem include-count /src/app/main/main.c 69 problem function-size /src/app/main/main.c:dumpstats() 102 -problem function-size /src/app/main/main.c:tor_init() 137 +problem function-size /src/app/main/main.c:tor_init() 101 problem function-size /src/app/main/main.c:sandbox_init_filter() 291 problem function-size /src/app/main/main.c:run_tor_main_loop() 105 problem function-size /src/app/main/ntmain.c:nt_service_install() 126 problem dependency-violation /src/core/crypto/hs_ntor.c 1 +problem dependency-violation /src/core/crypto/hs_ntor.h 1 problem dependency-violation /src/core/crypto/onion_crypto.c 5 problem dependency-violation /src/core/crypto/onion_fast.c 1 problem dependency-violation /src/core/crypto/onion_tap.c 3 problem dependency-violation /src/core/crypto/relay_crypto.c 9 -problem file-size /src/core/mainloop/connection.c 5569 +problem file-size /src/core/mainloop/connection.c 5577 problem include-count /src/core/mainloop/connection.c 62 problem function-size /src/core/mainloop/connection.c:connection_free_minimal() 185 problem function-size /src/core/mainloop/connection.c:connection_listener_new() 324 @@ -70,18 +75,20 @@ problem function-size /src/core/mainloop/connection.c:connection_handle_write_im problem function-size /src/core/mainloop/connection.c:assert_connection_ok() 143 problem dependency-violation /src/core/mainloop/connection.c 44 problem dependency-violation /src/core/mainloop/cpuworker.c 12 -problem include-count /src/core/mainloop/mainloop.c 63 +problem include-count /src/core/mainloop/mainloop.c 64 problem function-size /src/core/mainloop/mainloop.c:conn_close_if_marked() 108 problem function-size /src/core/mainloop/mainloop.c:run_connection_housekeeping() 123 -problem dependency-violation /src/core/mainloop/mainloop.c 49 +problem dependency-violation /src/core/mainloop/mainloop.c 50 problem dependency-violation /src/core/mainloop/mainloop_pubsub.c 1 problem dependency-violation /src/core/mainloop/mainloop_sys.c 1 problem dependency-violation /src/core/mainloop/netstatus.c 4 problem dependency-violation /src/core/mainloop/periodic.c 2 problem dependency-violation /src/core/or/address_set.c 1 +problem dependency-violation /src/core/or/cell_queue_st.h 1 problem file-size /src/core/or/channel.c 3487 problem dependency-violation /src/core/or/channel.c 9 -problem file-size /src/core/or/channel.h 780 +problem file-size /src/core/or/channel.h 781 +problem dependency-violation /src/core/or/channel.h 1 problem dependency-violation /src/core/or/channelpadding.c 6 problem function-size /src/core/or/channeltls.c:channel_tls_handle_var_cell() 160 problem function-size /src/core/or/channeltls.c:channel_tls_process_versions_cell() 170 @@ -96,15 +103,16 @@ problem function-size /src/core/or/circuitbuild.c:choose_good_exit_server_genera problem dependency-violation /src/core/or/circuitbuild.c 25 problem include-count /src/core/or/circuitlist.c 55 problem function-size /src/core/or/circuitlist.c:HT_PROTOTYPE() 109 -problem function-size /src/core/or/circuitlist.c:circuit_free_() 143 +problem function-size /src/core/or/circuitlist.c:circuit_free_() 146 problem function-size /src/core/or/circuitlist.c:circuit_find_to_cannibalize() 101 problem function-size /src/core/or/circuitlist.c:circuit_about_to_free() 120 problem function-size /src/core/or/circuitlist.c:circuits_handle_oom() 117 problem dependency-violation /src/core/or/circuitlist.c 19 +problem dependency-violation /src/core/or/circuitlist.h 1 problem function-size /src/core/or/circuitmux.c:circuitmux_set_policy() 109 problem function-size /src/core/or/circuitmux.c:circuitmux_attach_circuit() 113 problem dependency-violation /src/core/or/circuitmux_ewma.c 2 -problem file-size /src/core/or/circuitpadding.c 3096 +problem file-size /src/core/or/circuitpadding.c 3098 problem function-size /src/core/or/circuitpadding.c:circpad_machine_schedule_padding() 113 problem dependency-violation /src/core/or/circuitpadding.c 6 problem file-size /src/core/or/circuitpadding.h 813 @@ -119,33 +127,35 @@ problem function-size /src/core/or/circuituse.c:circuit_expire_building() 394 problem function-size /src/core/or/circuituse.c:circuit_log_ancient_one_hop_circuits() 126 problem function-size /src/core/or/circuituse.c:circuit_build_failed() 149 problem function-size /src/core/or/circuituse.c:circuit_launch_by_extend_info() 108 -problem function-size /src/core/or/circuituse.c:circuit_get_open_circ_or_launch() 352 +problem function-size /src/core/or/circuituse.c:circuit_get_open_circ_or_launch() 351 problem function-size /src/core/or/circuituse.c:connection_ap_handshake_attach_circuit() 244 -problem dependency-violation /src/core/or/circuituse.c 23 +problem dependency-violation /src/core/or/circuituse.c 24 problem function-size /src/core/or/command.c:command_process_create_cell() 156 problem function-size /src/core/or/command.c:command_process_relay_cell() 132 problem dependency-violation /src/core/or/command.c 8 -problem file-size /src/core/or/connection_edge.c 4596 +problem file-size /src/core/or/connection_edge.c 4640 problem include-count /src/core/or/connection_edge.c 65 problem function-size /src/core/or/connection_edge.c:connection_ap_expire_beginning() 117 -problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_rewrite() 191 +problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_rewrite() 193 problem function-size /src/core/or/connection_edge.c:connection_ap_handle_onion() 185 -problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_rewrite_and_attach() 421 +problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_rewrite_and_attach() 420 problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_send_begin() 111 problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_socks_resolved() 101 problem function-size /src/core/or/connection_edge.c:connection_exit_begin_conn() 185 problem function-size /src/core/or/connection_edge.c:connection_exit_connect() 102 problem dependency-violation /src/core/or/connection_edge.c 27 +problem dependency-violation /src/core/or/connection_edge.h 1 problem file-size /src/core/or/connection_or.c 3122 problem include-count /src/core/or/connection_or.c 51 problem function-size /src/core/or/connection_or.c:connection_or_group_set_badness_() 105 problem function-size /src/core/or/connection_or.c:connection_or_client_learned_peer_id() 142 problem function-size /src/core/or/connection_or.c:connection_or_compute_authenticate_cell_body() 231 problem dependency-violation /src/core/or/connection_or.c 20 -problem dependency-violation /src/core/or/dos.c 5 +problem dependency-violation /src/core/or/dos.c 6 problem dependency-violation /src/core/or/onion.c 2 problem file-size /src/core/or/or.h 1107 problem include-count /src/core/or/or.h 49 +problem dependency-violation /src/core/or/or.h 1 problem dependency-violation /src/core/or/or_periodic.c 1 problem file-size /src/core/or/policies.c 3249 problem function-size /src/core/or/policies.c:policy_summarize() 107 @@ -179,7 +189,7 @@ problem function-size /src/feature/client/addressmap.c:addressmap_rewrite() 109 problem function-size /src/feature/client/bridges.c:rewrite_node_address_for_bridge() 126 problem function-size /src/feature/client/circpathbias.c:pathbias_measure_close_rate() 108 problem function-size /src/feature/client/dnsserv.c:evdns_server_callback() 153 -problem file-size /src/feature/client/entrynodes.c 3824 +problem file-size /src/feature/client/entrynodes.c 3825 problem function-size /src/feature/client/entrynodes.c:entry_guards_upgrade_waiting_circuits() 155 problem function-size /src/feature/client/entrynodes.c:entry_guard_parse_from_state() 246 problem file-size /src/feature/client/entrynodes.h 639 @@ -190,11 +200,11 @@ problem function-size /src/feature/control/control.c:connection_control_process_ problem function-size /src/feature/control/control_auth.c:handle_control_authenticate() 186 problem function-size /src/feature/control/control_cmd.c:handle_control_extendcircuit() 150 problem function-size /src/feature/control/control_cmd.c:handle_control_add_onion() 256 -problem function-size /src/feature/control/control_cmd.c:add_onion_helper_keyarg() 116 +problem function-size /src/feature/control/control_cmd.c:add_onion_helper_keyarg() 118 problem function-size /src/feature/control/control_events.c:control_event_stream_status() 118 problem include-count /src/feature/control/control_getinfo.c 54 problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_misc() 108 -problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_dir() 302 +problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_dir() 297 problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_events() 234 problem function-size /src/feature/dirauth/bwauth.c:dirserv_read_measured_bandwidths() 121 problem file-size /src/feature/dirauth/dirvote.c 4700 @@ -215,8 +225,8 @@ problem function-size /src/feature/dircache/consdiffmgr.c:consdiffmgr_cleanup() problem function-size /src/feature/dircache/consdiffmgr.c:consdiffmgr_rescan_flavor_() 111 problem function-size /src/feature/dircache/consdiffmgr.c:consensus_diff_worker_threadfn() 132 problem function-size /src/feature/dircache/dircache.c:handle_get_current_consensus() 165 -problem function-size /src/feature/dircache/dircache.c:directory_handle_command_post() 119 -problem file-size /src/feature/dirclient/dirclient.c 3215 +problem function-size /src/feature/dircache/dircache.c:directory_handle_command_post() 124 +problem file-size /src/feature/dirclient/dirclient.c 3165 problem include-count /src/feature/dirclient/dirclient.c 51 problem function-size /src/feature/dirclient/dirclient.c:directory_get_from_dirserver() 126 problem function-size /src/feature/dirclient/dirclient.c:directory_initiate_request() 201 @@ -246,8 +256,8 @@ problem function-size /src/feature/hs/hs_descriptor.c:desc_encode_v3() 101 problem function-size /src/feature/hs/hs_descriptor.c:decrypt_desc_layer() 111 problem function-size /src/feature/hs/hs_descriptor.c:decode_introduction_point() 122 problem function-size /src/feature/hs/hs_descriptor.c:desc_decode_superencrypted_v3() 107 -problem function-size /src/feature/hs/hs_descriptor.c:desc_decode_encrypted_v3() 107 -problem file-size /src/feature/hs/hs_service.c 4116 +problem function-size /src/feature/hs/hs_descriptor.c:desc_decode_encrypted_v3() 109 +problem file-size /src/feature/hs/hs_service.c 4172 problem function-size /src/feature/keymgt/loadkey.c:ed_key_init_from_file() 326 problem function-size /src/feature/nodelist/authcert.c:trusted_dirs_load_certs_from_string() 123 problem function-size /src/feature/nodelist/authcert.c:authority_certs_fetch_missing() 295 @@ -260,7 +270,7 @@ problem function-size /src/feature/nodelist/node_select.c:router_pick_directory_ problem function-size /src/feature/nodelist/node_select.c:compute_weighted_bandwidths() 203 problem function-size /src/feature/nodelist/node_select.c:router_pick_trusteddirserver_impl() 112 problem function-size /src/feature/nodelist/nodelist.c:compute_frac_paths_available() 190 -problem file-size /src/feature/nodelist/routerlist.c 3241 +problem file-size /src/feature/nodelist/routerlist.c 3239 problem function-size /src/feature/nodelist/routerlist.c:router_rebuild_store() 148 problem function-size /src/feature/nodelist/routerlist.c:router_add_to_routerlist() 168 problem function-size /src/feature/nodelist/routerlist.c:routerlist_remove_old_routers() 121 @@ -269,8 +279,8 @@ problem function-size /src/feature/nodelist/routerlist.c:update_extrainfo_downlo problem function-size /src/feature/relay/dns.c:dns_resolve_impl() 131 problem function-size /src/feature/relay/dns.c:configure_nameservers() 161 problem function-size /src/feature/relay/dns.c:evdns_callback() 108 -problem file-size /src/feature/relay/router.c 3522 -problem include-count /src/feature/relay/router.c 56 +problem file-size /src/feature/relay/router.c 3520 +problem include-count /src/feature/relay/router.c 57 problem function-size /src/feature/relay/router.c:init_keys() 252 problem function-size /src/feature/relay/router.c:get_my_declared_family() 114 problem function-size /src/feature/relay/router.c:router_build_fresh_unsigned_routerinfo() 136 @@ -279,14 +289,14 @@ problem function-size /src/feature/relay/routerkeys.c:load_ed_keys() 294 problem function-size /src/feature/rend/rendcache.c:rend_cache_store_v2_desc_as_client() 190 problem function-size /src/feature/rend/rendclient.c:rend_client_send_introduction() 219 problem function-size /src/feature/rend/rendcommon.c:rend_encode_v2_descriptors() 221 -problem function-size /src/feature/rend/rendmid.c:rend_mid_establish_intro_legacy() 104 +problem function-size /src/feature/rend/rendmid.c:rend_mid_establish_intro_legacy() 105 problem function-size /src/feature/rend/rendparse.c:rend_parse_v2_service_descriptor() 181 problem function-size /src/feature/rend/rendparse.c:rend_parse_introduction_points() 129 -problem file-size /src/feature/rend/rendservice.c 4511 +problem file-size /src/feature/rend/rendservice.c 4522 problem function-size /src/feature/rend/rendservice.c:rend_service_prune_list_impl_() 107 problem function-size /src/feature/rend/rendservice.c:rend_config_service() 162 problem function-size /src/feature/rend/rendservice.c:rend_service_load_auth_keys() 178 -problem function-size /src/feature/rend/rendservice.c:rend_service_receive_introduction() 330 +problem function-size /src/feature/rend/rendservice.c:rend_service_receive_introduction() 334 problem function-size /src/feature/rend/rendservice.c:rend_service_parse_intro_for_v3() 111 problem function-size /src/feature/rend/rendservice.c:rend_service_decrypt_intro() 112 problem function-size /src/feature/rend/rendservice.c:rend_service_intro_has_opened() 126 @@ -310,7 +320,7 @@ problem function-size /src/lib/net/address.c:tor_addr_compare_masked() 110 problem function-size /src/lib/net/inaddr.c:tor_inet_pton() 107 problem function-size /src/lib/net/socketpair.c:tor_ersatz_socketpair() 102 problem function-size /src/lib/osinfo/uname.c:get_uname() 116 -problem function-size /src/lib/process/process_unix.c:process_unix_exec() 220 +problem function-size /src/lib/process/process_unix.c:process_unix_exec() 213 problem function-size /src/lib/process/process_win32.c:process_win32_exec() 151 problem function-size /src/lib/process/process_win32.c:process_win32_create_pipe() 109 problem function-size /src/lib/process/restrict.c:set_max_file_descriptors() 102 @@ -324,12 +334,3 @@ problem function-size /src/tools/tor-gencert.c:parse_commandline() 111 problem function-size /src/tools/tor-resolve.c:build_socks5_resolve_request() 102 problem function-size /src/tools/tor-resolve.c:do_resolve() 171 problem function-size /src/tools/tor-resolve.c:main() 112 - -problem dependency-violation /scripts/maint/practracker/testdata/a.c 3 -problem dependency-violation /scripts/maint/practracker/testdata/header.h 3 -problem dependency-violation /src/core/crypto/hs_ntor.h 1 -problem dependency-violation /src/core/or/cell_queue_st.h 1 -problem dependency-violation /src/core/or/channel.h 1 -problem dependency-violation /src/core/or/circuitlist.h 1 -problem dependency-violation /src/core/or/connection_edge.h 1 -problem dependency-violation /src/core/or/or.h 1 diff --git a/scripts/maint/practracker/practracker.py b/scripts/maint/practracker/practracker.py index f6aac9d15e..5f26d28cea 100755 --- a/scripts/maint/practracker/practracker.py +++ b/scripts/maint/practracker/practracker.py @@ -147,7 +147,7 @@ HEADER="""\ # file-size -- a .c file of more than {MAX_FILE_SIZE} lines, or a .h # file with more than {MAX_H_FILE_SIZE} lines. # include-count -- a .c file with more than {MAX_INCLUDE_COUNT} #includes, - or a .h file with more than {MAX_H_INCLUDE_COUNT} #includes. +# or a .h file with more than {MAX_H_INCLUDE_COUNT} #includes. # dependency-violation -- a file includes a header that it should # not, according to an advisory .may_include file. # diff --git a/scripts/maint/practracker/util.py b/scripts/maint/practracker/util.py index df629110c2..db02a983f8 100644 --- a/scripts/maint/practracker/util.py +++ b/scripts/maint/practracker/util.py @@ -35,6 +35,12 @@ def get_tor_c_files(tor_topdir, include_dirs=None): continue if filename in EXCLUDE_FILES: continue + # Avoid editor temporary files + bname = os.path.basename(filename) + if bname.startswith("."): + continue + if bname.startswith("#"): + continue full_path = os.path.join(root,filename) diff --git a/scripts/maint/rename_c_identifier.py b/scripts/maint/rename_c_identifier.py new file mode 100755 index 0000000000..213172d913 --- /dev/null +++ b/scripts/maint/rename_c_identifier.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2001 Matej Pfajfar. +# Copyright (c) 2001-2004, Roger Dingledine. +# Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. +# Copyright (c) 2007-2019, The Tor Project, Inc. +# See LICENSE for licensing information + +""" +Helpful script to replace one or more C identifiers, and optionally +generate a commit message explaining what happened. +""" + +import argparse +import fileinput +import os +import re +import shlex +import subprocess +import sys +import tempfile + +TOPDIR = "src" + + +def is_c_file(fn): + """ + Return true iff fn is the name of a C file. + + >>> is_c_file("a/b/module.c") + True + >>> is_c_file("a/b/module.h") + True + >>> is_c_file("a/b/module.c~") + False + >>> is_c_file("a/b/.module.c") + False + >>> is_c_file("a/b/module.cpp") + False + """ + fn = os.path.split(fn)[1] + if fn.startswith("."): + return False + ext = os.path.splitext(fn)[1] + return ext in {".c", ".h", ".i", ".inc"} + + +def list_c_files(topdir=TOPDIR): + """ + Use git to list all the C files under version control. + + >>> lst = list(list_c_files()) + >>> "src/core/mainloop/mainloop.c" in lst + True + >>> "src/core/mainloop/twiddledeedoo.c" in lst + False + >>> "micro-revision.i" in lst + False + """ + proc = subprocess.Popen( + ["git", "ls-tree", "--name-only", "-r", "HEAD", topdir], + stdout=subprocess.PIPE, + encoding="utf-8") + for line in proc.stdout.readlines(): + line = line.strip() + if is_c_file(line): + yield line + + +class Rewriter: + """ + A rewriter applies a series of word-by-word replacements, in + sequence. Replacements only happen at "word boundaries", + as determined by the \\b regular expression marker. + + ("A word is defined as a sequence of alphanumeric or underscore + characters", according to the documentation.) + + >>> R = Rewriter([("magic", "secret"), ("words", "codes")]) + >>> R.apply("The magic words are rambunctious bluejay") + 'The secret codes are rambunctious bluejay' + >>> R.apply("The magical words are rambunctious bluejay") + 'The magical codes are rambunctious bluejay' + >>> R.get_count() + 3 + + """ + + def __init__(self, replacements): + """Make a new Rewriter. Takes a sequence of pairs of + (from_id, to_id), where from_id is an identifier to replace, + and to_id is its replacement. + """ + self._patterns = [] + for id1, id2 in replacements: + pat = re.compile(r"\b{}\b".format(re.escape(id1))) + self._patterns.append((pat, id2)) + + self._count = 0 + + def apply(self, line): + """Return `line` as transformed by this rewriter.""" + for pat, ident in self._patterns: + line, count = pat.subn(ident, line) + self._count += count + return line + + def get_count(self): + """Return the number of identifiers that this rewriter has + rewritten.""" + return self._count + + +def rewrite_files(files, rewriter): + """ + Apply `rewriter` to every file in `files`, replacing those files + with their rewritten contents. + """ + for line in fileinput.input(files, inplace=True): + sys.stdout.write(rewriter.apply(line)) + + +def make_commit_msg(pairs, no_verify): + """Return a commit message to explain what was replaced by the provided + arguments. + """ + script = ["./scripts/maint/rename_c_identifier.py"] + for id1, id2 in pairs: + qid1 = shlex.quote(id1) + qid2 = shlex.quote(id2) + script.append(" {} {}".format(qid1, qid2)) + script = " \\\n".join(script) + + if len(pairs) == 1: + line1 = "Rename {} to {}".format(*pairs[0]) + else: + line1 = "Replace several C identifiers." + + msg = """\ +{} + +This is an automated commit, generated by this command: + +{} +""".format(line1, script) + + if no_verify: + msg += """ +It was generated with --no-verify, so it probably breaks some commit hooks. +The commiter should be sure to fix them up in a subsequent commit. +""" + + return msg + + +def commit(pairs, no_verify=False): + """Try to commit the current git state, generating the commit message as + appropriate. If `no_verify` is True, pass the --no-verify argument to + git commit. + """ + args = [] + if no_verify: + args.append("--no-verify") + + # We have to use a try block to delete the temporary file here, since we + # are using tempfile with delete=False. We have to use delete=False, + # since otherwise we are not guaranteed to be able to give the file to + # git for it to open. + fname = None + try: + with tempfile.NamedTemporaryFile(mode="w", delete=False) as f: + fname = f.name + f.write(make_commit_msg(pairs, no_verify)) + s = subprocess.run(["git", "commit", "-a", "-F", fname, "--edit"]+args) + if s.returncode != 0 and not no_verify: + print('"git commit" failed. Maybe retry with --no-verify?', + file=sys.stderr) + revert_changes() + return False + finally: + os.unlink(fname) + + return True + + +def any_uncommitted_changes(): + """Return True if git says there are any uncommitted changes in the current + working tree; false otherwise. + """ + s = subprocess.run(["git", "diff-index", "--quiet", "HEAD"]) + return s.returncode != 0 + + +DESC = "Replace one identifier with another throughout our source." +EXAMPLES = """\ +Examples: + + rename_c_identifier.py set_ctrl_id set_controller_id + (Replaces every occurrence of "set_ctrl_id" with "set_controller_id".) + + rename_c_identifier.py --commit set_ctrl_id set_controller_id + (As above, but also generate a git commit with an appropriate message.) + + rename_c_identifier.py a b c d + (Replace "a" with "b", and "c" with "d".)""" + + +def revert_changes(): + """Tell git to revert all the changes in the current working tree. + """ + print('Reverting changes.', file=sys.stderr) + subprocess.run(["git", "checkout", "--quiet", TOPDIR]) + + +def main(argv): + import argparse + parser = argparse.ArgumentParser(description=DESC, epilog=EXAMPLES, + # prevent re-wrapping the examples + formatter_class=argparse.RawDescriptionHelpFormatter) + + parser.add_argument("--commit", action='store_true', + help="Generate a Git commit.") + parser.add_argument("--no-verify", action='store_true', + help="Tell Git not to run its pre-commit hooks.") + parser.add_argument("from_id", type=str, help="Original identifier") + parser.add_argument("to_id", type=str, help="New identifier") + parser.add_argument("more", type=str, nargs=argparse.REMAINDER, + help="Additional identifier pairs") + + args = parser.parse_args(argv[1:]) + + if len(args.more) % 2 != 0: + print("I require an even number of identifiers.", file=sys.stderr) + return 1 + + if any_uncommitted_changes(): + print("Uncommitted changes found. Not running.", file=sys.stderr) + return 1 + + pairs = [] + print("renaming {} to {}".format(args.from_id, args.to_id), file=sys.stderr) + pairs.append((args.from_id, args.to_id)) + for idx in range(0, len(args.more), 2): + id1 = args.more[idx] + id2 = args.more[idx+1] + print("renaming {} to {}".format(id1, id2)) + pairs.append((id1, id2)) + + rewriter = Rewriter(pairs) + + rewrite_files(list_c_files(), rewriter) + + print("Replaced {} identifiers".format(rewriter.get_count()), + file=sys.stderr) + + if args.commit: + commit(pairs, args.no_verify) + + +if __name__ == '__main__': + main(sys.argv) diff --git a/scripts/maint/update_versions.py b/scripts/maint/update_versions.py index 8067f2c6c8..706a93b6a2 100755 --- a/scripts/maint/update_versions.py +++ b/scripts/maint/update_versions.py @@ -95,7 +95,7 @@ def update_file(fname, replace_on_change(fname, have_changed) # Find out our version -with open("configure.ac") as f: +with open(P("configure.ac")) as f: version = find_version(f) # If we have no version, we can't proceed. |