diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/git/git-merge-forward.sh | 281 | ||||
| -rwxr-xr-x | scripts/git/git-push-all.sh | 219 | ||||
| -rwxr-xr-x | scripts/git/pre-commit.git-hook | 4 | ||||
| -rwxr-xr-x | scripts/maint/checkShellScripts.sh | 64 | ||||
| -rw-r--r-- | scripts/maint/practracker/README | 21 | ||||
| -rw-r--r-- | scripts/maint/practracker/exceptions.txt | 26 | ||||
| -rwxr-xr-x | scripts/maint/practracker/includes.py | 2 | ||||
| -rwxr-xr-x | scripts/maint/practracker/practracker.py | 19 |
8 files changed, 542 insertions, 94 deletions
diff --git a/scripts/git/git-merge-forward.sh b/scripts/git/git-merge-forward.sh index ba29983284..cbd2f3c3bd 100755 --- a/scripts/git/git-merge-forward.sh +++ b/scripts/git/git-merge-forward.sh @@ -1,9 +1,23 @@ #!/usr/bin/env bash -############################## -# Configuration (change me!) # -############################## - +# Usage: git-merge-forward.sh -n -t <test-branch-prefix> -u +# arguments: +# -n: dry run mode +# -t: test branch mode: create new branches from the commits checked +# out in each maint directory. Call these branches prefix_029, +# prefix_035, ... , prefix_master. +# -u: in test branch mode, if a prefix_* branch exists, skip creating +# that branch. Use after a merge error, to restart the merge +# forward at the first unmerged branch. +# env vars: +# See the Configuration section for env vars and their default values. + +################# +# Configuration # +################# + +# Don't change this configuration - set the env vars in your .profile +# # The general setup that is suggested here is: # # GIT_PATH = /home/<user>/git/ @@ -21,27 +35,55 @@ TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"} # The worktrees location (directory). TOR_WKT_NAME=${TOR_WKT_NAME:-"tor-wkt"} -######################### -# End of configuration. # -######################### +########################## +# Git branches to manage # +########################## + +# The branches and worktrees need to be modified when there is a new branch, +# and when an old branch is no longer supported. # Configuration of the branches that needs merging. The values are in order: -# (1) Branch name that we merge onto. -# (2) Branch name to merge from. In other words, this is merge into (1) -# (3) Full path of the git worktree. +# (0) current maint/release branch name +# (1) previous maint/release name to merge into (0) +# (only used in merge forward mode) +# (2) Full path of the git worktree +# (3) current branch suffix +# (maint branches only, only used in test branch mode) +# (4) previous test branch suffix to merge into (3) +# (maint branches only, only used in test branch mode) # -# As an example: -# $ cd <PATH/TO/WORKTREE> (3) -# $ git checkout maint-0.3.5 (1) +# Merge forward example: +# $ cd <PATH/TO/WORKTREE> (2) +# $ git checkout maint-0.3.5 (0) # $ git pull -# $ git merge maint-0.3.4 (2) +# $ git merge maint-0.3.4 (1) +# +# Test branch example: +# $ cd <PATH/TO/WORKTREE> (2) +# $ git checkout -b ticket99999_035 (3) +# $ git checkout maint-0.3.5 (0) +# $ git pull +# $ git checkout ticket99999_035 +# $ git merge maint-0.3.5 +# $ git merge ticket99999_034 (4) # # First set of arrays are the maint-* branch and then the release-* branch. # New arrays need to be in the WORKTREE= array else they aren't considered. -MAINT_035=( "maint-0.3.5" "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.5" ) -MAINT_040=( "maint-0.4.0" "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.0" ) -MAINT_041=( "maint-0.4.1" "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.1" ) -MAINT_MASTER=( "master" "maint-0.4.1" "$GIT_PATH/$TOR_MASTER_NAME" ) +# +# Only used in test branch mode +# There is no previous branch to merge forward, so the second and fifth items +# must be blank ("") +MAINT_029_TB=( "maint-0.2.9" "" "$GIT_PATH/$TOR_WKT_NAME/maint-0.2.9" \ + "_029" "") +# Used in maint/release merge and test branch modes +MAINT_035=( "maint-0.3.5" "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.5" \ + "_035" "_029") +MAINT_040=( "maint-0.4.0" "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.0" \ + "_040" "_035") +MAINT_041=( "maint-0.4.1" "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.1" \ + "_041" "_040") +MAINT_MASTER=( "master" "maint-0.4.1" "$GIT_PATH/$TOR_MASTER_NAME" \ + "_master" "_041") RELEASE_029=( "release-0.2.9" "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/release-0.2.9" ) RELEASE_035=( "release-0.3.5" "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.5" ) @@ -55,6 +97,7 @@ ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME" # SC2034 -- shellcheck thinks that these are unused. We know better. ACTUALLY_THESE_ARE_USED=<<EOF +${MAINT_029_TB[0]} ${MAINT_035[0]} ${MAINT_040[0]} ${MAINT_041[0]} @@ -65,31 +108,88 @@ ${RELEASE_040[0]} ${RELEASE_041[0]} EOF -########################## -# Git Worktree to manage # -########################## +####################### +# Argument processing # +####################### + +# Controlled by the -n option. The dry run option will just output the command +# that would have been executed for each worktree. +DRY_RUN=0 + +# Controlled by the -t <test-branch-prefix> option. The test branch base +# name option makes git-merge-forward.sh create new test branches: +# <tbbn>_029, <tbbn>_035, ... , <tbbn>_master, and merge forward. +TEST_BRANCH_PREFIX= + +# Controlled by the -u option. The use existing option checks for existing +# branches with the <test-branch-prefix>, and checks them out, rather than +# creating a new branch. +USE_EXISTING=0 + +while getopts "nt:u" opt; do + case "$opt" in + n) DRY_RUN=1 + echo " *** DRY RUN MODE ***" + ;; + t) TEST_BRANCH_PREFIX="$OPTARG" + echo " *** CREATING TEST BRANCHES: ${TEST_BRANCH_PREFIX}_nnn ***" + ;; + u) USE_EXISTING=1 + echo " *** USE EXISTING TEST BRANCHES MODE ***" + ;; + *) + exit 1 + ;; + esac +done + +########################### +# Git worktrees to manage # +########################### + +if [ -z "$TEST_BRANCH_PREFIX" ]; then + + # maint/release merge mode + # + # List of all worktrees to work on. All defined above. Ordering is important. + # Always the maint-* branch BEFORE then the release-*. + WORKTREE=( + RELEASE_029[@] + + MAINT_035[@] + RELEASE_035[@] + + MAINT_040[@] + RELEASE_040[@] + + MAINT_041[@] + RELEASE_041[@] + + MAINT_MASTER[@] + ) + +else -# List of all worktrees to work on. All defined above. Ordering is important. -# Always the maint-* branch BEFORE then the release-*. -WORKTREE=( - RELEASE_029[@] + # Test branch mode: merge to maint only, and create a new branch for 0.2.9 + WORKTREE=( + MAINT_029_TB[@] - MAINT_035[@] - RELEASE_035[@] + MAINT_035[@] - MAINT_040[@] - RELEASE_040[@] + MAINT_040[@] - MAINT_041[@] - RELEASE_041[@] + MAINT_041[@] + + MAINT_MASTER[@] + ) + +fi - MAINT_MASTER[@] -) COUNT=${#WORKTREE[@]} -# Controlled by the -n option. The dry run option will just output the command -# that would have been executed for each worktree. -DRY_RUN=0 +############# +# Constants # +############# # Control characters CNRM=$'\x1b[0;0m' # Clear color @@ -127,7 +227,7 @@ function validate_ret # Switch to the given branch name. function switch_branch { - local cmd="git checkout $1" + local cmd="git checkout '$1'" printf " %s Switching branch to %s..." "$MARKER" "$1" if [ $DRY_RUN -eq 0 ]; then msg=$( eval "$cmd" 2>&1 ) @@ -137,6 +237,45 @@ function switch_branch fi } +# Checkout a new branch with the given branch name. +function new_branch +{ + local cmd="git checkout -b '$1'" + printf " %s Creating new branch %s..." "$MARKER" "$1" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Switch to an existing branch, or checkout a new branch with the given +# branch name. +function switch_or_new_branch +{ + local cmd="git rev-parse --verify '$1'" + if [ $DRY_RUN -eq 0 ]; then + # Call switch_branch if there is a branch, or new_branch if there is not + msg=$( eval "$cmd" 2>&1 ) + RET=$? + if [ $RET -eq 0 ]; then + # Branch: (commit id) + switch_branch "$1" + elif [ $RET -eq 128 ]; then + # Not a branch: "fatal: Needed a single revision" + new_branch "$1" + else + # Unexpected return value + validate_ret $RET "$msg" + fi + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}, then depending on the result:" + switch_branch "$1" + new_branch "$1" + fi +} + # Pull the given branch name. function pull_branch { @@ -150,10 +289,10 @@ function pull_branch fi } -# Merge the given branch name ($2) into the current branch ($1). +# Merge the given branch name ($1) into the current branch ($2). function merge_branch { - local cmd="git merge --no-edit $1" + local cmd="git merge --no-edit '$1'" printf " %s Merging branch %s into %s..." "$MARKER" "$1" "$2" if [ $DRY_RUN -eq 0 ]; then msg=$( eval "$cmd" 2>&1 ) @@ -166,7 +305,7 @@ function merge_branch # Pull the given branch name. function merge_branch_origin { - local cmd="git merge --ff-only origin/$1" + local cmd="git merge --ff-only 'origin/$1'" printf " %s Merging branch origin/%s..." "$MARKER" "$1" if [ $DRY_RUN -eq 0 ]; then msg=$( eval "$cmd" 2>&1 ) @@ -203,16 +342,6 @@ function fetch_origin # Entry point # ############### -while getopts "n" opt; do - case "$opt" in - n) DRY_RUN=1 - echo " *** DRY DRUN MODE ***" - ;; - *) - ;; - esac -done - # First, fetch the origin. goto_repo "$ORIGIN_PATH" fetch_origin @@ -222,15 +351,57 @@ for ((i=0; i<COUNT; i++)); do current=${!WORKTREE[$i]:0:1} previous=${!WORKTREE[$i]:1:1} repo_path=${!WORKTREE[$i]:2:1} + # default to merge forward mode + test_current= + test_previous= + target_current="$current" + target_previous="$previous" + if [ "$TEST_BRANCH_PREFIX" ]; then + test_current_suffix=${!WORKTREE[$i]:3:1} + test_current=${TEST_BRANCH_PREFIX}${test_current_suffix} + # the current test branch, if present, or maint/release branch, if not + target_current="$test_current" + test_previous_suffix=${!WORKTREE[$i]:4:1} + if [ "$test_previous_suffix" ]; then + test_previous=${TEST_BRANCH_PREFIX}${test_previous_suffix} + # the previous test branch, if present, or maint/release branch, if not + target_previous="$test_previous" + fi + fi - printf "%s Handling branch \\n" "$MARKER" "${BYEL}$current${CNRM}" + printf "%s Handling branch \\n" "$MARKER" "${BYEL}$target_current${CNRM}" # Go into the worktree to start merging. goto_repo "$repo_path" - # Checkout the current branch + if [ "$test_current" ]; then + if [ $USE_EXISTING -eq 0 ]; then + # Create a test branch from the currently checked-out branch/commit + # Fail if it already exists + new_branch "$test_current" + else + # Switch if it exists, or create if it does not + switch_or_new_branch "$test_current" + fi + fi + # Checkout the current maint/release branch switch_branch "$current" - # Update the current branch with an origin merge to get the latest. + # Update the current maint/release branch with an origin merge to get the + # latest updates merge_branch_origin "$current" - # Merge the previous branch. Ex: merge maint-0.2.5 into maint-0.2.9. - merge_branch "$previous" "$current" + if [ "$test_current" ]; then + # Checkout the test branch + switch_branch "$test_current" + # Merge the updated maint branch into the test branch + merge_branch "$current" "$test_current" + fi + # Merge the previous branch into the target branch + # Merge Forward Example: + # merge maint-0.2.9 into maint-0.3.5. + # Test Branch Example: + # merge bug99999_029 into bug99999_035. + # Skip the merge if the previous branch does not exist + # (there's nothing to merge forward into the oldest test branch) + if [ "$target_previous" ]; then + merge_branch "$target_previous" "$target_current" + fi done diff --git a/scripts/git/git-push-all.sh b/scripts/git/git-push-all.sh index 469d6fe570..8e49e81b9d 100755 --- a/scripts/git/git-push-all.sh +++ b/scripts/git/git-push-all.sh @@ -1,19 +1,133 @@ #!/usr/bin/env bash -# Usage: git-push-all.sh -# env vars: TOR_UPSTREAM_REMOTE_NAME=upstream TOR_PUSH_DELAY=0 -# options: --no-atomic --dry-run (any other git push option) -# -# TOR_PUSH_DELAY pushes the master and maint branches separately, so that CI -# runs in a sensible order. -# push --atomic is the default when TOR_PUSH_DELAY=0, and for release branches. +# Usage: git-push-all.sh -t <test-branch-prefix> -r <remote-name> -s +# -- <git-opts> +# arguments: +# -t: test branch mode: Push test branches, rather than maint and +# release branches. Pushes the branches called prefix_029, +# prefix_035, ... , prefix_master. +# -r: push to remote-name, rather than $TOR_UPSTREAM_REMOTE_NAME. +# -s: push branches whose tips match upstream maint, release, or +# master branches. The default is to skip these branches. Use +# -s when testing for CI environment failures with old code. +# --: pass any other arguments to git, rather than the script. +# env vars: +# TOR_GIT_PUSH: the git push command and arguments +# TOR_UPSTREAM_REMOTE_NAME: the default upstream, overridden by -r +# TOR_PUSH_DELAY: pushes the master and maint branches separately, +# so that CI runs in a sensible order. +# TOR_PUSH_SAME: push branches whose tips match upstream maint, +# release, or master branches. Inverted by -s. +# See the Configuration section for env var default values. +# git-opts: +# --no-atomic --dry-run (and any other git push option) set -e +################# +# Configuration # +################# + +# Don't change this configuration - set the env vars in your .profile +# +# git push command and default arguments +GIT_PUSH=${TOR_GIT_PUSH:-"git push --atomic"} # The upstream remote which git.torproject.org/tor.git points to. -UPSTREAM_REMOTE=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"} +DEFAULT_UPSTREAM_REMOTE=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"} +# Push to a different upstream remote using -r <remote-name> +UPSTREAM_REMOTE=${DEFAULT_UPSTREAM_REMOTE} # Add a delay between pushes, so CI runs on the most important branches first PUSH_DELAY=${TOR_PUSH_DELAY:-0} +# Push (1) or skip (0) test branches that are the same as an upstream +# maint/master branch. Push if you are testing that the CI environment still +# works on old code, skip if you are testing new code in the branch. +# Default: skip unchanged branches. +# Inverted by the -s option. +PUSH_SAME=${TOR_PUSH_SAME:-0} + +####################### +# Argument processing # +####################### + +# Controlled by the -t <test-branch-prefix> option. The test branch base +# name option makes git-merge-forward.sh create new test branches: +# <tbbn>_029, <tbbn>_035, ... , <tbbn>_master, and merge forward. +TEST_BRANCH_PREFIX= + +while getopts ":r:st:" opt; do + case "$opt" in + r) UPSTREAM_REMOTE="$OPTARG" + echo " *** PUSHING TO REMOTE: ${UPSTREAM_REMOTE} ***" + shift + shift + OPTIND=$((OPTIND - 2)) + ;; + s) PUSH_SAME=$((! PUSH_SAME)) + if [ "$PUSH_SAME" -eq 0 ]; then + echo " *** SKIPPING UNCHANGED TEST BRANCHES ***" + else + echo " *** PUSHING UNCHANGED TEST BRANCHES ***" + fi + shift + OPTIND=$((OPTIND - 1)) + ;; + t) TEST_BRANCH_PREFIX="$OPTARG" + echo " *** PUSHING TEST BRANCHES: ${TEST_BRANCH_PREFIX}_nnn ***" + shift + shift + OPTIND=$((OPTIND - 2)) + ;; + *) + # Assume we're done with script arguments, + # and git push will handle the option + break + ;; + esac +done + +# getopts doesn't allow "-" as an option character, +# so we have to handle -- manually +if [ "$1" = "--" ]; then + shift +fi + +echo "Calling $GIT_PUSH" "$@" "<branches>" + +if [ "$TEST_BRANCH_PREFIX" ]; then + if [ "$UPSTREAM_REMOTE" = "${TOR_UPSTREAM_REMOTE_NAME:-upstream}" ]; then + echo "Pushing test branches ${TEST_BRANCH_PREFIX}_nnn to " \ + "$UPSTREAM_REMOTE is not allowed." + echo "Usage: $0 -r <remote-name> -t <test-branch-prefix> <git-opts>" + exit 1 + fi +fi + +################################ +# Git upstream remote branches # +################################ + +DEFAULT_UPSTREAM_BRANCHES= +if [ "$DEFAULT_UPSTREAM_REMOTE" != "$UPSTREAM_REMOTE" ]; then + DEFAULT_UPSTREAM_BRANCHES=$(echo \ + "$DEFAULT_UPSTREAM_REMOTE"/master \ + "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.4.1 \ + "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.4.0 \ + "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.3.5 \ + "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.2.9 \ + ) +fi + +UPSTREAM_BRANCHES=$(echo \ + "$UPSTREAM_REMOTE"/master \ + "$UPSTREAM_REMOTE"/{release,maint}-0.4.1 \ + "$UPSTREAM_REMOTE"/{release,maint}-0.4.0 \ + "$UPSTREAM_REMOTE"/{release,maint}-0.3.5 \ + "$UPSTREAM_REMOTE"/{release,maint}-0.2.9 \ + ) + +######################## +# Git branches to push # +######################## PUSH_BRANCHES=$(echo \ master \ @@ -23,29 +137,98 @@ PUSH_BRANCHES=$(echo \ {release,maint}-0.2.9 \ ) +if [ -z "$TEST_BRANCH_PREFIX" ]; then + + # maint/release push mode + # + # List of branches to push. Ordering is not important. + PUSH_BRANCHES=$(echo \ + master \ + {release,maint}-0.4.1 \ + {release,maint}-0.4.0 \ + {release,maint}-0.3.5 \ + {release,maint}-0.2.9 \ + ) +else + + # Test branch mode: merge to maint only, and create a new branch for 0.2.9 + # + # List of branches to push. Ordering is not important. + PUSH_BRANCHES=" \ + ${TEST_BRANCH_PREFIX}_master \ + ${TEST_BRANCH_PREFIX}_041 \ + ${TEST_BRANCH_PREFIX}_040 \ + ${TEST_BRANCH_PREFIX}_035 \ + ${TEST_BRANCH_PREFIX}_029 \ + " +fi + +############### +# Entry point # +############### + +# Skip the test branches that are the same as the upstream branches +if [ "$PUSH_SAME" -eq 0 ] && [ "$TEST_BRANCH_PREFIX" ]; then + NEW_PUSH_BRANCHES= + for b in $PUSH_BRANCHES; do + PUSH_COMMIT=$(git rev-parse "$b") + SKIP_UPSTREAM= + for u in $DEFAULT_UPSTREAM_BRANCHES $UPSTREAM_BRANCHES; do + UPSTREAM_COMMIT=$(git rev-parse "$u") + if [ "$PUSH_COMMIT" = "$UPSTREAM_COMMIT" ]; then + SKIP_UPSTREAM="$u" + fi + done + if [ "$SKIP_UPSTREAM" ]; then + printf "Skipping unchanged: %s remote: %s\n" \ + "$b" "$SKIP_UPSTREAM" + else + if [ "$NEW_PUSH_BRANCHES" ]; then + NEW_PUSH_BRANCHES="${NEW_PUSH_BRANCHES} ${b}" + else + NEW_PUSH_BRANCHES="${b}" + fi + fi + done + PUSH_BRANCHES=${NEW_PUSH_BRANCHES} +fi + if [ "$PUSH_DELAY" -le 0 ]; then echo "Pushing $PUSH_BRANCHES" # We know that there are no spaces in any branch within $PUSH_BRANCHES, so # it is safe to use it unquoted. (This also applies to the other shellcheck # exceptions below.) # + # Push all the branches at the same time # shellcheck disable=SC2086 - git push --atomic "$@" "$UPSTREAM_REMOTE" $PUSH_BRANCHES + $GIT_PUSH "$@" "$UPSTREAM_REMOTE" $PUSH_BRANCHES else + # Push the branches in optimal CI order, with a delay between each push PUSH_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\n" | sort -V) MASTER_BRANCH=$(echo "$PUSH_BRANCHES" | tr " " "\n" | grep master) - MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\n" | grep maint) - RELEASE_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\n" | grep release | \ - tr "\n" " ") - printf "Pushing with %ss delays, so CI runs in this order:\n%s\n%s\n%s\n" \ - "$PUSH_DELAY" "$MASTER_BRANCH" "$MAINT_BRANCHES" "$RELEASE_BRANCHES" - git push "$@" "$UPSTREAM_REMOTE" "$MASTER_BRANCH" + if [ -z "$TEST_BRANCH_PREFIX" ]; then + MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\n" | grep maint) + RELEASE_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\n" | grep release | \ + tr "\n" " ") + printf "Pushing with %ss delays, so CI runs in this order:\n%s\n%s\n%s\n" \ + "$PUSH_DELAY" "$MASTER_BRANCH" "$MAINT_BRANCHES" "$RELEASE_BRANCHES" + else + # Actually test branches based on maint branches + MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\n" | grep -v master) + printf "Pushing with %ss delays, so CI runs in this order:\n%s\n%s\n" \ + "$PUSH_DELAY" "$MASTER_BRANCH" "$MAINT_BRANCHES" + # No release branches + RELEASE_BRANCHES= + fi + $GIT_PUSH "$@" "$UPSTREAM_REMOTE" "$MASTER_BRANCH" sleep "$PUSH_DELAY" # shellcheck disable=SC2086 for b in $MAINT_BRANCHES; do - git push "$@" "$UPSTREAM_REMOTE" $b + $GIT_PUSH "$@" "$UPSTREAM_REMOTE" "$b" sleep "$PUSH_DELAY" done - # shellcheck disable=SC2086 - git push --atomic "$@" "$UPSTREAM_REMOTE" $RELEASE_BRANCHES + if [ "$RELEASE_BRANCHES" ]; then + # shellcheck disable=SC2086 + $GIT_PUSH "$@" "$UPSTREAM_REMOTE" $RELEASE_BRANCHES + fi fi diff --git a/scripts/git/pre-commit.git-hook b/scripts/git/pre-commit.git-hook index b2a1847a2b..1c381ec60a 100755 --- a/scripts/git/pre-commit.git-hook +++ b/scripts/git/pre-commit.git-hook @@ -53,3 +53,7 @@ if [ -e "${PT_DIR}/practracker.py" ]; then fi fi fi + +if [ -e scripts/maint/checkShellScripts.sh ]; then + scripts/maint/checkShellScripts.sh +fi diff --git a/scripts/maint/checkShellScripts.sh b/scripts/maint/checkShellScripts.sh new file mode 100755 index 0000000000..02d1275c48 --- /dev/null +++ b/scripts/maint/checkShellScripts.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Tor Project, Inc. +# See LICENSE for license information +# +# checkShellScripts.sh +# -------------------- +# If shellcheck is installed, check all the shell scripts that we can fix. + +set -e + +# Only run this script if shellcheck is installed +# command echoes the path to shellcheck, which is a useful diagnostic log +if ! command -v shellcheck; then + printf "%s: Install shellcheck to check shell scripts.\\n" "$0" + exit 0 +fi + +# Some platforms don't have realpath +if command -v realpath ; then + HERE=$(dirname "$(realpath "$0")") +else + HERE=$(dirname "$0") + if [ ! -d "$HERE" ]; then + HERE=$(dirname "$PWD/$0") + fi +fi +TOPLEVEL=$(dirname "$(dirname "$HERE")") + +# Check we actually have a tor/src directory +if [ ! -d "$TOPLEVEL/src" ]; then + printf "Error: Couldn't find src directory in expected location: %s\\n" \ + "$TOPLEVEL/src" +fi + +# Check *.sh scripts, but ignore the ones that we can't fix +find "$TOPLEVEL" \ + -name "*.sh" \ + -path "$TOPLEVEL/contrib/*" \ + -path "$TOPLEVEL/doc/*" \ + -path "$TOPLEVEL/scripts/*" \ + -path "$TOPLEVEL/src/*" \ + -not -path "$TOPLEVEL/src/ext/*" \ + -not -path "$TOPLEVEL/src/rust/registry/*" \ + -exec shellcheck {} + + +# Check scripts that aren't named *.sh +if [ -d "$TOPLEVEL/scripts/test" ]; then + shellcheck \ + "$TOPLEVEL/scripts/test/cov-diff" \ + "$TOPLEVEL/scripts/test/coverage" +fi +if [ -e \ + "$TOPLEVEL/contrib/dirauth-tools/nagios-check-tor-authority-cert" \ + ]; then + shellcheck \ + "$TOPLEVEL/contrib/dirauth-tools/nagios-check-tor-authority-cert" +fi +if [ -e "$TOPLEVEL/contrib/client-tools/torify" ]; then + shellcheck "$TOPLEVEL/contrib/client-tools/torify" +fi +if [ -d "$TOPLEVEL/scripts/git" ]; then + shellcheck "$TOPLEVEL/scripts/git/"*.git-hook +fi diff --git a/scripts/maint/practracker/README b/scripts/maint/practracker/README new file mode 100644 index 0000000000..d978b39806 --- /dev/null +++ b/scripts/maint/practracker/README @@ -0,0 +1,21 @@ +Practracker is a simple python tool that keeps track of places where +our code is ugly, and tries to warn us about new ones or ones that +get worse. + +Right now, practracker looks for the following kinds of +best-practices violations: + + .c files greater than 3000 lines long + .h files greater than 500 lines long + .c files with more than 50 includes + .h files with more than 15 includes + + All files that include a local header not listed in a .may_include + file in the same directory, when that .may_include file has an + "!advisory" marker. + +The list of current violations is tracked in exceptions.txt; slight +deviations of the current exceptions cause warnings, whereas large +ones cause practracker to fail. + +For usage information, run "practracker.py --help". diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index f0306ebeba..2e676d9045 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -44,7 +44,6 @@ problem function-size /src/app/config/config.c:parse_dir_authority_line() 150 problem function-size /src/app/config/config.c:parse_dir_fallback_line() 101 problem function-size /src/app/config/config.c:parse_port_config() 446 problem function-size /src/app/config/config.c:parse_ports() 168 -problem function-size /src/app/config/config.c:getinfo_helper_config() 113 problem file-size /src/app/config/or_options_st.h 1112 problem include-count /src/app/main/main.c 68 problem function-size /src/app/main/main.c:dumpstats() 102 @@ -81,8 +80,8 @@ problem dependency-violation /src/core/mainloop/netstatus.c 4 problem dependency-violation /src/core/mainloop/periodic.c 2 problem dependency-violation /src/core/or/address_set.c 1 problem file-size /src/core/or/channel.c 3487 -problem file-size /src/core/or/channel.h 780 problem dependency-violation /src/core/or/channel.c 9 +problem file-size /src/core/or/channel.h 780 problem dependency-violation /src/core/or/channelpadding.c 6 problem function-size /src/core/or/channeltls.c:channel_tls_handle_var_cell() 160 problem function-size /src/core/or/channeltls.c:channel_tls_process_versions_cell() 170 @@ -105,10 +104,10 @@ problem dependency-violation /src/core/or/circuitlist.c 19 problem function-size /src/core/or/circuitmux.c:circuitmux_set_policy() 109 problem function-size /src/core/or/circuitmux.c:circuitmux_attach_circuit() 113 problem dependency-violation /src/core/or/circuitmux_ewma.c 2 -problem file-size /src/core/or/circuitpadding.c 3043 -problem function-size /src/core/or/circuitpadding.c:circpad_machine_schedule_padding() 107 -problem file-size /src/core/or/circuitpadding.h 809 +problem file-size /src/core/or/circuitpadding.c 3096 +problem function-size /src/core/or/circuitpadding.c:circpad_machine_schedule_padding() 113 problem dependency-violation /src/core/or/circuitpadding.c 6 +problem file-size /src/core/or/circuitpadding.h 813 problem function-size /src/core/or/circuitpadding_machines.c:circpad_machine_relay_hide_intro_circuits() 103 problem function-size /src/core/or/circuitpadding_machines.c:circpad_machine_client_hide_rend_circuits() 112 problem dependency-violation /src/core/or/circuitpadding_machines.c 1 @@ -142,19 +141,19 @@ problem include-count /src/core/or/connection_or.c 51 problem function-size /src/core/or/connection_or.c:connection_or_group_set_badness_() 105 problem function-size /src/core/or/connection_or.c:connection_or_client_learned_peer_id() 142 problem function-size /src/core/or/connection_or.c:connection_or_compute_authenticate_cell_body() 231 -problem file-size /src/core/or/or.h 1103 -problem include-count /src/core/or/or.h 49 problem dependency-violation /src/core/or/connection_or.c 20 problem dependency-violation /src/core/or/dos.c 5 problem dependency-violation /src/core/or/onion.c 2 +problem file-size /src/core/or/or.h 1107 +problem include-count /src/core/or/or.h 49 problem dependency-violation /src/core/or/or_periodic.c 1 problem file-size /src/core/or/policies.c 3249 problem function-size /src/core/or/policies.c:policy_summarize() 107 problem dependency-violation /src/core/or/policies.c 14 problem function-size /src/core/or/protover.c:protover_all_supported() 117 -problem function-size /src/core/or/relay.c:circuit_receive_relay_cell() 127 -problem file-size /src/core/or/relay.c 3263 problem dependency-violation /src/core/or/reasons.c 2 +problem file-size /src/core/or/relay.c 3264 +problem function-size /src/core/or/relay.c:circuit_receive_relay_cell() 127 problem function-size /src/core/or/relay.c:relay_send_command_from_edge_() 109 problem function-size /src/core/or/relay.c:connection_ap_process_end_not_open() 192 problem function-size /src/core/or/relay.c:connection_edge_process_relay_cell_not_open() 137 @@ -237,18 +236,19 @@ problem function-size /src/feature/dirparse/parsecommon.c:get_next_token() 158 problem function-size /src/feature/dirparse/routerparse.c:router_parse_entry_from_string() 554 problem function-size /src/feature/dirparse/routerparse.c:extrainfo_parse_entry_from_string() 208 problem function-size /src/feature/hibernate/hibernate.c:accounting_parse_options() 109 -problem function-size /src/feature/hs/hs_cell.c:hs_cell_build_establish_intro() 113 +problem function-size /src/feature/hs/hs_cell.c:hs_cell_build_establish_intro() 115 problem function-size /src/feature/hs/hs_cell.c:hs_cell_parse_introduce2() 152 problem function-size /src/feature/hs/hs_client.c:send_introduce1() 103 problem function-size /src/feature/hs/hs_client.c:hs_config_client_authorization() 107 problem function-size /src/feature/hs/hs_common.c:hs_get_responsible_hsdirs() 102 +problem function-size /src/feature/hs/hs_config.c:config_service_v3() 107 problem function-size /src/feature/hs/hs_config.c:config_generic_service() 138 problem function-size /src/feature/hs/hs_descriptor.c:desc_encode_v3() 101 problem function-size /src/feature/hs/hs_descriptor.c:decrypt_desc_layer() 105 problem function-size /src/feature/hs/hs_descriptor.c:decode_introduction_point() 122 problem function-size /src/feature/hs/hs_descriptor.c:desc_decode_superencrypted_v3() 107 problem function-size /src/feature/hs/hs_descriptor.c:desc_decode_encrypted_v3() 107 -problem file-size /src/feature/hs/hs_service.c 4109 +problem file-size /src/feature/hs/hs_service.c 4116 problem function-size /src/feature/keymgt/loadkey.c:ed_key_init_from_file() 326 problem function-size /src/feature/nodelist/authcert.c:trusted_dirs_load_certs_from_string() 123 problem function-size /src/feature/nodelist/authcert.c:authority_certs_fetch_missing() 295 @@ -261,7 +261,7 @@ problem function-size /src/feature/nodelist/node_select.c:router_pick_directory_ problem function-size /src/feature/nodelist/node_select.c:compute_weighted_bandwidths() 203 problem function-size /src/feature/nodelist/node_select.c:router_pick_trusteddirserver_impl() 112 problem function-size /src/feature/nodelist/nodelist.c:compute_frac_paths_available() 190 -problem file-size /src/feature/nodelist/routerlist.c 3239 +problem file-size /src/feature/nodelist/routerlist.c 3241 problem function-size /src/feature/nodelist/routerlist.c:router_rebuild_store() 148 problem function-size /src/feature/nodelist/routerlist.c:router_add_to_routerlist() 168 problem function-size /src/feature/nodelist/routerlist.c:routerlist_remove_old_routers() 121 @@ -280,7 +280,7 @@ problem function-size /src/feature/relay/routerkeys.c:load_ed_keys() 294 problem function-size /src/feature/rend/rendcache.c:rend_cache_store_v2_desc_as_client() 190 problem function-size /src/feature/rend/rendclient.c:rend_client_send_introduction() 219 problem function-size /src/feature/rend/rendcommon.c:rend_encode_v2_descriptors() 221 -problem function-size /src/feature/rend/rendmid.c:rend_mid_establish_intro_legacy() 103 +problem function-size /src/feature/rend/rendmid.c:rend_mid_establish_intro_legacy() 104 problem function-size /src/feature/rend/rendparse.c:rend_parse_v2_service_descriptor() 181 problem function-size /src/feature/rend/rendparse.c:rend_parse_introduction_points() 129 problem file-size /src/feature/rend/rendservice.c 4511 diff --git a/scripts/maint/practracker/includes.py b/scripts/maint/practracker/includes.py index fcc527c983..397439b4ef 100755 --- a/scripts/maint/practracker/includes.py +++ b/scripts/maint/practracker/includes.py @@ -212,7 +212,7 @@ def consider_include_rules(fname, f): log_sorted_levels = False def walk_c_files(topdir="src"): - """Run through all c and h files under topdir, looking for + """Run through all .c and .h files under topdir, looking for include-rule violations. Yield those violations.""" for dirpath, dirnames, fnames in os.walk(topdir): diff --git a/scripts/maint/practracker/practracker.py b/scripts/maint/practracker/practracker.py index b280a76765..78003fb786 100755 --- a/scripts/maint/practracker/practracker.py +++ b/scripts/maint/practracker/practracker.py @@ -7,7 +7,8 @@ Go through the various .c files and collect metrics about them. If the metrics violate some of our best practices and they are not found in the optional exceptions file, then log a problem about them. -We currently do metrics about file size, function size and number of includes. +We currently do metrics about file size, function size and number of includes, +for C source files and headers. practracker.py should be run with its second argument pointing to the Tor top-level source directory like this: @@ -143,8 +144,12 @@ HEADER="""\ # # There are three kinds of problems that we recognize right now: # function-size -- a function of more than {MAX_FUNCTION_SIZE} lines. -# file-size -- a file of more than {MAX_FILE_SIZE} lines. -# include-count -- a file with more than {MAX_INCLUDE_COUNT} #includes. +# file-size -- a .c file of more than {MAX_FILE_SIZE} lines, or a .h +# file with more than {MAX_H_FILE_SIZE} lines. +# include-count -- a .c file with more than {MAX_INCLUDE_COUNT} #includes, + or a .h file with more than {MAX_H_INCLUDE_COUNT} #includes. +# dependency-violation -- a file includes a header that it should +# not, according to an advisory .may_include file. # # Each line below represents a single exception that practracker should # _ignore_. Each line has four parts: @@ -184,13 +189,13 @@ def main(argv): parser.add_argument("--terse", action="store_true", help="Do not emit helpful instructions.") parser.add_argument("--max-h-file-size", default=MAX_H_FILE_SIZE, - help="Maximum lines per .H file") + help="Maximum lines per .h file") parser.add_argument("--max-h-include-count", default=MAX_H_INCLUDE_COUNT, - help="Maximum includes per .H file") + help="Maximum includes per .h file") parser.add_argument("--max-file-size", default=MAX_FILE_SIZE, - help="Maximum lines per C file") + help="Maximum lines per .c file") parser.add_argument("--max-include-count", default=MAX_INCLUDE_COUNT, - help="Maximum includes per C file") + help="Maximum includes per .c file") parser.add_argument("--max-function-size", default=MAX_FUNCTION_SIZE, help="Maximum lines per function") parser.add_argument("--max-dependency-violations", default=MAX_DEP_VIOLATIONS, |