diff options
Diffstat (limited to 'scripts')
32 files changed, 933 insertions, 322 deletions
diff --git a/scripts/codegen/fuzzing_include_am.py b/scripts/codegen/fuzzing_include_am.py index aa3ba49a73..ae50563074 100755 --- a/scripts/codegen/fuzzing_include_am.py +++ b/scripts/codegen/fuzzing_include_am.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Future imports for Python 2.7, mandatory in 3.0 from __future__ import division diff --git a/scripts/codegen/gen_server_ciphers.py b/scripts/codegen/gen_server_ciphers.py index 3b77952243..8c88e54a13 100755 --- a/scripts/codegen/gen_server_ciphers.py +++ b/scripts/codegen/gen_server_ciphers.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright 2014-2019, The Tor Project, Inc # See LICENSE for licensing information diff --git a/scripts/codegen/get_mozilla_ciphers.py b/scripts/codegen/get_mozilla_ciphers.py index 165105736a..ff01dd8719 100755 --- a/scripts/codegen/get_mozilla_ciphers.py +++ b/scripts/codegen/get_mozilla_ciphers.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # coding=utf-8 # Copyright 2011-2019, The Tor Project, Inc # original version by Arturo Filastò diff --git a/scripts/codegen/makedesc.py b/scripts/codegen/makedesc.py index af926a6438..48d1d31a02 100644 --- a/scripts/codegen/makedesc.py +++ b/scripts/codegen/makedesc.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright 2014-2019, The Tor Project, Inc. # See LICENSE for license information diff --git a/scripts/git/git-install-tools.sh b/scripts/git/git-install-tools.sh new file mode 100755 index 0000000000..ef8623a018 --- /dev/null +++ b/scripts/git/git-install-tools.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash + +SCRIPT_NAME=$(basename "$0") +SCRIPTS_DIR=$(dirname "$0") + +TOOL_NAMES=(push-all pull-all merge-forward list-tor-branches) + +function usage() +{ + echo "$SCRIPT_NAME [-h] [-n] [-v] [-f] <all|hooks|tools|aliases>" + echo + echo " flags:" + echo " -h: show this help text" + echo " -n: dry-run" + echo " -v: verbose mode" + echo " -f: force-install even if \$TOR_DEVTOOLS_DIR looks fishy" + echo + echo " modes:" + echo " hooks: install git hooks in this repository." + echo " tools: install scripts in \$TOR_DEVTOOLS_DIR" + echo " aliases: set up global git aliases for git tools in \$TOR_DEVTOOLS_DIR" + echo " all: all of the above." +} + +INSTALL_HOOKS=0 +INSTALL_TOOLS=0 +INSTALL_ALIASES=0 + +DRY_RUN=0 +VERBOSE=0 +FORCE=0 + +while getopts "hnfv" opt; do + case "$opt" in + h) usage + exit 0 + ;; + n) DRY_RUN=1 + ;; + v) VERBOSE=1 + ;; + f) FORCE=1 + ;; + *) echo + usage + exit 1 + ;; + esac +done + +for item in "${@:$OPTIND}"; do + case "$item" in + hooks) INSTALL_HOOKS=1 + ;; + tools) INSTALL_TOOLS=1 + ;; + aliases) INSTALL_ALIASES=1 + ;; + all) INSTALL_HOOKS=1 + INSTALL_TOOLS=1 + INSTALL_ALIASES=1 + ;; + *) echo "Unrecognized mode '$item'" + usage + exit 1 + ;; + esac +done + +if [[ $VERBOSE = 1 ]]; then + function note() + { + echo "$@" + } +else + function note() + { + true + } +fi + +function fail() +{ + echo "$@" 1>&2 + exit 1 +} + +if [[ $INSTALL_HOOKS = 0 && $INSTALL_TOOLS = 0 && $INSTALL_ALIASES = 0 ]]; then + echo "Nothing to do. Try $SCRIPT_NAME -h for a list of commands." + exit 0 +fi + +if [[ $INSTALL_TOOLS = 1 || $INSTALL_ALIASES = 1 ]]; then + if [[ -z "$TOR_DEVTOOLS_DIR" ]] ; then + fail "\$TOR_DEVTOOLS_DIR was not set." + fi + note "Checking whether \$TOR_DEVTOOLS_DIR ($TOR_DEVTOOLS_DIR) is a git repo..." + GITDIR=$(cd "$TOR_DEVTOOLS_DIR" && git rev-parse --git-dir 2>/dev/null) + note "GITDIR is $GITDIR" + if [[ -n "$GITDIR" ]] ; then + cat <<EOF +You have asked me to install to \$TOR_DEVTOOLS_DIR ($TOR_DEVTOOLS_DIR). +That is inside a git repository, so you might not want to install there: +depending on what you pull or push, you might find yourself giving somebody +else write access to your scripts. I think you should just use ~/bin or +something. +EOF + + echo + if [[ "$FORCE" = 1 ]] ; then + echo "I will install anyway, since you said '-f'." + else + echo "I will not install. You can tell me -f if you are really sure." + exit 1 + fi + else + note "It was not." + fi +fi + +if [[ ! -d "$SCRIPTS_DIR" || ! -e "$SCRIPTS_DIR/git-push-all.sh" ]]; then + fail "Couldn't find scripts in '$SCRIPTS_DIR'" +fi + +if [[ $DRY_RUN = 1 ]]; then + echo "** DRY RUN **" + RUN="echo >>" +else + RUN= +fi + +set -e + +# ====================================================================== +if [[ $INSTALL_HOOKS = 1 ]]; then + HOOKS_DIR=$(git rev-parse --git-path hooks) + + note "Looking for hooks directory" + + if [[ -z "$HOOKS_DIR" || ! -d "$HOOKS_DIR" ]]; then + fail "Couldn't find git hooks directory." + fi + + note "Found hooks directory in $HOOKS_DIR" + + note "Installing hooks" + for fn in "$SCRIPTS_DIR"/*.git-hook; do + name=$(basename "$fn") + $RUN install --backup "$fn" "${HOOKS_DIR}/${name%.git-hook}" + done +fi + + +# ====================================================================== +if [[ $INSTALL_TOOLS = 1 ]]; then + note "Installing tools." + note "Looking for \$TOR_DEVTOOLS_DIR ($TOR_DEVTOOLS_DIR)" + + if [[ ! -d "$TOR_DEVTOOLS_DIR" ]]; then + note "Creating directory" + $RUN mkdir -p "$TOR_DEVTOOLS_DIR" + fi + + note "Copying scripts" + for tool in "${TOOL_NAMES[@]}"; do + $RUN install --backup "${SCRIPTS_DIR}/git-${tool}.sh" "${TOR_DEVTOOLS_DIR}/" + done +fi + +# ====================================================================== +if [[ $INSTALL_ALIASES = 1 ]]; then + note "Installing aliases." + note "Looking for \$TOR_DEVTOOLS_DIR ($TOR_DEVTOOLS_DIR)" + + note "Checking for ${TOR_DEVTOOLS_DIR}/git-push-all.sh" + if [[ ! -x "${TOR_DEVTOOLS_DIR}/git-push-all.sh" ]]; then + if [[ $DRY_RUN = 0 ]]; then + fail "Could not find scripts in \$TOR_DEVTOOLS_DIR" + fi + fi + + note "Setting aliases" + for tool in "${TOOL_NAMES[@]}"; do + $RUN git config --global "alias.$tool" \!"${TOR_DEVTOOLS_DIR}/git-${tool}.sh" + done + +fi + +note Done. diff --git a/scripts/git/git-list-tor-branches.sh b/scripts/git/git-list-tor-branches.sh new file mode 100755 index 0000000000..d6b30f064f --- /dev/null +++ b/scripts/git/git-list-tor-branches.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash + +# Script to be used by other git scripts, and provide a single place +# that lists our supported branches. To change which branches are +# supported, look at the end of the file that says 'edit here'. + +SCRIPT_NAME=$(basename "$0") + +function usage() +{ + echo "$SCRIPT_NAME [-h] [-l|-s|-b|-m] [-R]" + echo + echo " arguments:" + echo " -h: show this help text" + echo + echo " -l: list the active tor branches (default)" + echo " -s: list the suffixes to be used with the active tor branches" + echo " -b: write bash code setting WORKTREE to an array of ( branch path ) arrays" + echo " -m: write bash code setting WORKTREE to an array of" + echo " ( branch parent path suffix parent_suffix ) arrays" + echo + echo " -R: omit release branches." +} + +# list : just a list of branch names. +# branch_path : For git-setup-dirs.sh and git-pull-all.sh +# suffix: write a list of suffixes. +# merge: branch, upstream, path, suffix, upstream suffix. +mode="list" +skip_release_branches="no" + +while getopts "hblmsR" opt ; do + case "$opt" in + h) usage + exit 0 + ;; + b) mode="branch_path" + ;; + l) mode="list" + ;; + s) mode="suffix" + ;; + m) mode="merge" + ;; + R) skip_release_branches="yes" + ;; + *) echo "Unknown option" + exit 1 + ;; + esac +done + +all_branch_vars=() + +prev_maint_branch="" +prev_maint_suffix="" + +branch() { + # The name of the branch. (Supplied by caller) Ex: maint-0.4.3 + brname="$1" + + # The name of the branch with no dots. Ex: maint-043 + brname_nodots="${brname//./}" + # The name of the branch with no dots, and _ instead of -. Ex: maint_043 + brname_nodots_uscore="${brname_nodots//-/_}" + # Name to use for a variable to represent the branch. Ex: MAINT_043 + varname="${brname_nodots_uscore^^}" + + is_maint="no" + + # suffix: a suffix to place at the end of branches we generate with respect + # to this branch. Ex: _043 + + # location: where the branch can be found. + + if [[ "$brname" == "master" ]]; then + suffix="_master" + location="\$GIT_PATH/\$TOR_MASTER_NAME" + elif [[ "$brname" =~ ^maint- ]]; then + suffix="_${brname_nodots#maint-}" + location="\$GIT_PATH/\$TOR_WKT_NAME/$brname" + is_maint="yes" + elif [[ "$brname" =~ ^release- ]]; then + suffix="_r${brname_nodots#release-}" + location="\$GIT_PATH/\$TOR_WKT_NAME/$brname" + + if [[ "$skip_release_branches" = "yes" ]]; then + return + fi + else + echo "Unrecognized branch type '${brname}'" >&2 + exit 1 + fi + + all_branch_vars+=("$varname") + + # Now emit the per-branch information + if [[ "$mode" == "branch_path" ]]; then + echo "${varname}=( \"$brname\" \"$location\" )" + elif [[ "$mode" == "merge" ]]; then + echo "${varname}=( \"$brname\" \"$prev_maint_branch\" \"$location\" \"$suffix\" \"$prev_maint_suffix\" )" + elif [[ "$mode" == "list" ]]; then + echo "$brname" + elif [[ "$mode" == "suffix" ]]; then + echo "$suffix" + else + echo "unknown mode $mode" >&2 + exit 1 + fi + + if [[ "$is_maint" == "yes" ]]; then + prev_maint_branch="$brname" + prev_maint_suffix="$suffix" + fi +} + +finish() { + if [[ "$mode" == branch_path ]] || [[ "$mode" == merge ]]; then + echo "WORKTREE=(" + for v in "${all_branch_vars[@]}"; do + echo " ${v}[@]" + done + echo ")" + elif [[ "$mode" == list ]] || [[ "$mode" == suffix ]]; then + # nothing to do + : + else + echo "unknown mode $mode" >&2 + exit 1 + fi +} + +# ============================== +# EDIT HERE +# ============================== +# List of all branches. These must be in order, from oldest to newest, with +# maint before release. + +branch maint-0.3.5 +branch release-0.3.5 + +branch maint-0.4.1 +branch release-0.4.1 + +branch maint-0.4.2 +branch release-0.4.2 + +branch maint-0.4.3 +branch release-0.4.3 + +branch master + +finish diff --git a/scripts/git/git-merge-forward.sh b/scripts/git/git-merge-forward.sh index bbc5047cb7..7c72f8478d 100755 --- a/scripts/git/git-merge-forward.sh +++ b/scripts/git/git-merge-forward.sh @@ -91,41 +91,11 @@ TOR_WKT_NAME=${TOR_WKT_NAME:-"tor-wkt"} # But it's the earliest maint branch, so we don't merge forward into it. # Since we don't merge forward into it, the second and fifth items must be # blank (""). -MAINT_035_TB=( "maint-0.3.5" "" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.5" \ - "_035" "") -# Used in maint/release merge and test branch modes -MAINT_040=( "maint-0.4.0" "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.0" \ - "_040" "_035") -MAINT_041=( "maint-0.4.1" "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.1" \ - "_041" "_040") -MAINT_042=( "maint-0.4.2" "maint-0.4.1" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.2" \ - "_042" "_041") -MAINT_MASTER=( "master" "maint-0.4.2" "$GIT_PATH/$TOR_MASTER_NAME" \ - "_master" "_042") - -RELEASE_035=( "release-0.3.5" "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.5" ) -RELEASE_040=( "release-0.4.0" "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.0" ) -RELEASE_041=( "release-0.4.1" "maint-0.4.1" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.1" ) -RELEASE_042=( "release-0.4.2" "maint-0.4.2" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.2" ) - -# The master branch path has to be the main repository thus contains the + # origin that will be used to fetch the updates. All the worktrees are created # from that repository. ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME" -# SC2034 -- shellcheck thinks that these are unused. We know better. -ACTUALLY_THESE_ARE_USED=<<EOF -${MAINT_035_TB[0]} -${MAINT_040[0]} -${MAINT_041[0]} -${MAINT_042[0]} -${MAINT_MASTER[0]} -${RELEASE_035[0]} -${RELEASE_040[0]} -${RELEASE_041[0]} -${RELEASE_042[0]} -EOF - ####################### # Argument processing # ####################### @@ -170,49 +140,16 @@ done # Git worktrees to manage # ########################### +set -e if [ -z "$TEST_BRANCH_PREFIX" ]; then - # maint/release merge mode - # - # List of all worktrees to merge forward into. All defined above. - # Ordering is important. Always the maint-* branch BEFORE the release-*. - WORKTREE=( - # We don't merge forward into MAINT_035_TB[@], because it's the earliest - # maint branch - RELEASE_035[@] - - MAINT_040[@] - RELEASE_040[@] - - MAINT_041[@] - RELEASE_041[@] - - MAINT_042[@] - RELEASE_042[@] - - MAINT_MASTER[@] - ) - + eval "$(git-list-tor-branches.sh -m)" + # Remove first element: we don't merge forward into it. + WORKTREE=( "${WORKTREE[@]:1}" ) else - - # Test branch mode: base test branches on maint branches only - # - # List of all worktrees to create test branches from. All defined above. - # Ordering is important. All maint-* branches, including the earliest one. - WORKTREE=( - # We want a test branch based on the earliest maint branch - MAINT_035_TB[@] - - MAINT_040[@] - - MAINT_041[@] - - MAINT_042[@] - - MAINT_MASTER[@] - ) - + eval "$(git-list-tor-branches.sh -m -R)" fi +set +e COUNT=${#WORKTREE[@]} diff --git a/scripts/git/git-pull-all.sh b/scripts/git/git-pull-all.sh index c8d115da01..7f82eda296 100755 --- a/scripts/git/git-pull-all.sh +++ b/scripts/git/git-pull-all.sh @@ -2,7 +2,7 @@ SCRIPT_NAME=$(basename "$0") -function usage() +usage() { echo "$SCRIPT_NAME [-h] [-n]" echo @@ -47,66 +47,15 @@ TOR_WKT_NAME=${TOR_WKT_NAME:-"tor-wkt"} # Git branches to manage # ########################## -# Configuration of the branches that need pulling. The values are in order: -# (1) Branch name to pull (update). -# (2) Full path of the git worktree. -# -# As an example: -# $ cd <PATH/TO/WORKTREE> (3) -# $ git checkout maint-0.3.5 (1) -# $ git pull -# -# First set of arrays are the maint-* branch and then the release-* branch. -# New arrays need to be in the WORKTREE= array else they aren't considered. -MAINT_035=( "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.5" ) -MAINT_040=( "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.0" ) -MAINT_041=( "maint-0.4.1" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.1" ) -MAINT_042=( "maint-0.4.2" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.2" ) -MAINT_MASTER=( "master" "$GIT_PATH/$TOR_MASTER_NAME" ) - -RELEASE_035=( "release-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.5" ) -RELEASE_040=( "release-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.0" ) -RELEASE_041=( "release-0.4.1" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.1" ) -RELEASE_042=( "release-0.4.2" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.2" ) +set -e +eval "$(git-list-tor-branches.sh -b)" +set +e # The master branch path has to be the main repository thus contains the # origin that will be used to fetch the updates. All the worktrees are created # from that repository. ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME" -# SC2034 -- shellcheck thinks that these are unused. We know better. -ACTUALLY_THESE_ARE_USED=<<EOF -${MAINT_035[0]} -${MAINT_040[0]} -${MAINT_041[0]} -${MAINT_042[0]} -${MAINT_MASTER[0]} -${RELEASE_035[0]} -${RELEASE_040[0]} -${RELEASE_041[0]} -${RELEASE_042[0]} -EOF - -########################### -# Git worktrees to manage # -########################### - -# List of all worktrees to pull. All defined above. Ordering is not important. -WORKTREE=( - MAINT_035[@] - RELEASE_035[@] - - MAINT_040[@] - RELEASE_040[@] - - MAINT_041[@] - RELEASE_041[@] - - MAINT_042[@] - RELEASE_042[@] - - MAINT_MASTER[@] -) COUNT=${#WORKTREE[@]} ####################### diff --git a/scripts/git/git-push-all.sh b/scripts/git/git-push-all.sh index 0abddc8023..558ea8d01c 100755 --- a/scripts/git/git-push-all.sh +++ b/scripts/git/git-push-all.sh @@ -168,63 +168,42 @@ echo "Calling $GIT_PUSH" "$@" "<branches>" # Git upstream remote branches # ################################ +set -e DEFAULT_UPSTREAM_BRANCHES= if [ "$DEFAULT_UPSTREAM_REMOTE" != "$UPSTREAM_REMOTE" ]; then - DEFAULT_UPSTREAM_BRANCHES=$(echo \ - "$DEFAULT_UPSTREAM_REMOTE"/master \ - "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.4.2 \ - "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.4.1 \ - "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.4.0 \ - "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.3.5 \ - ) + for br in $(git-list-tor-branches.sh -l); do + DEFAULT_UPSTREAM_BRANCHES="${DEFAULT_UPSTREAM_BRANCHES} ${DEFAULT_UPSTREAM_REMOTE}/${br}" + done fi -UPSTREAM_BRANCHES=$(echo \ - "$UPSTREAM_REMOTE"/master \ - "$UPSTREAM_REMOTE"/{release,maint}-0.4.2 \ - "$UPSTREAM_REMOTE"/{release,maint}-0.4.1 \ - "$UPSTREAM_REMOTE"/{release,maint}-0.4.0 \ - "$UPSTREAM_REMOTE"/{release,maint}-0.3.5 \ - ) +UPSTREAM_BRANCHES= +for br in $(git-list-tor-branches.sh -l); do + UPSTREAM_BRANCHES="${UPSTREAM_BRANCHES} ${UPSTREAM_REMOTE}/${br}" +done ######################## # Git branches to push # ######################## -PUSH_BRANCHES=$(echo \ - master \ - {release,maint}-0.4.2 \ - {release,maint}-0.4.1 \ - {release,maint}-0.4.0 \ - {release,maint}-0.3.5 \ - ) - if [ -z "$TEST_BRANCH_PREFIX" ]; then # maint/release push mode: push all branches. # # List of branches to push. Ordering is not important. - PUSH_BRANCHES=$(echo \ - master \ - {release,maint}-0.4.2 \ - {release,maint}-0.4.1 \ - {release,maint}-0.4.0 \ - {release,maint}-0.3.5 \ - ) + PUSH_BRANCHES="$(git-list-tor-branches.sh -l)" else # Test branch push mode: push test branches, based on each maint branch. # # List of branches to push. Ordering is not important. - PUSH_BRANCHES=" \ - ${TEST_BRANCH_PREFIX}_master \ - ${TEST_BRANCH_PREFIX}_042 \ - ${TEST_BRANCH_PREFIX}_041 \ - ${TEST_BRANCH_PREFIX}_040 \ - ${TEST_BRANCH_PREFIX}_035 \ - " + PUSH_BRANCHES="" + for suffix in $(git-list-tor-branches.sh -s -R); do + PUSH_BRANCHES="${PUSH_BRANCHES} ${TEST_BRANCH_PREFIX}${suffix}" + done fi +set +e + ############### # Entry point # ############### diff --git a/scripts/git/git-setup-dirs.sh b/scripts/git/git-setup-dirs.sh index 20a148204a..1f61eb8b83 100755 --- a/scripts/git/git-setup-dirs.sh +++ b/scripts/git/git-setup-dirs.sh @@ -90,41 +90,15 @@ GITHUB_PUSH=${TOR_GITHUB_PUSH:-"No_Pushing_To_GitHub"} # The branches and worktrees need to be modified when there is a new branch, # and when an old branch is no longer supported. -# Configuration of the branches that needs merging. The values are in order: -# (0) current maint/release branch name -# (1) Full path of the git worktree -# -# First set of arrays are the maint-* branch and then the release-* branch. -# New arrays need to be in the WORKTREE= array else they aren't considered. -MAINT_035=( "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.5" ) -MAINT_040=( "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.0" ) -MAINT_041=( "maint-0.4.1" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.1" ) -MAINT_042=( "maint-0.4.2" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.2" ) -MAINT_MASTER=( "master" "$GIT_PATH/$TOR_MASTER_NAME" ) - -RELEASE_035=( "release-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.5" ) -RELEASE_040=( "release-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.0" ) -RELEASE_041=( "release-0.4.1" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.1" ) -RELEASE_042=( "release-0.4.2" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.2" ) +set -e +eval "$(git-list-tor-branches.sh -b)" +set +e # The master branch path has to be the main repository thus contains the # origin that will be used to fetch the updates. All the worktrees are created # from that repository. ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME" -# SC2034 -- shellcheck thinks that these are unused. We know better. -ACTUALLY_THESE_ARE_USED=<<EOF -${MAINT_035[0]} -${MAINT_040[0]} -${MAINT_041[0]} -${MAINT_042[0]} -${MAINT_MASTER[0]} -${RELEASE_035[0]} -${RELEASE_040[0]} -${RELEASE_041[0]} -${RELEASE_042[0]} -EOF - ####################### # Argument processing # ####################### @@ -161,22 +135,6 @@ done # Git worktrees to manage # ########################### -WORKTREE=( - MAINT_035[@] - RELEASE_035[@] - - MAINT_040[@] - RELEASE_040[@] - - MAINT_041[@] - RELEASE_041[@] - - MAINT_042[@] - RELEASE_042[@] - - MAINT_MASTER[@] -) - COUNT=${#WORKTREE[@]} ############# diff --git a/scripts/git/pre-push.git-hook b/scripts/git/pre-push.git-hook index 7b06f3734d..efa45b9860 100755 --- a/scripts/git/pre-push.git-hook +++ b/scripts/git/pre-push.git-hook @@ -26,7 +26,11 @@ z40=0000000000000000000000000000000000000000 upstream_name=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"} +# The working directory workdir=$(git rev-parse --show-toplevel) +# The .git directory +# If $workdir is a worktree, then $gitdir is not $workdir/.git +gitdir=$(git rev-parse --git-dir) cd "$workdir" || exit 1 @@ -58,7 +62,8 @@ do fi # Call the pre-commit hook for the common checks, if it is executable - if [ -x scripts/git/pre-commit.git-hook ]; then + pre_commit=${gitdir}/hooks/pre-commit + if [ -x "$pre_commit" ]; then # Only check the files newly modified in this branch CHECK_FILTER="git diff --name-only --diff-filter=ACMR $range" # Use the appropriate owned tor source list to filter the changed @@ -81,7 +86,7 @@ do # We want word splitting here, because file names are space # separated # shellcheck disable=SC2086 - if ! scripts/git/pre-commit.git-hook $CHECK_FILES ; then + if ! "$pre_commit" $CHECK_FILES ; then exit 1 fi fi diff --git a/scripts/maint/annotate_ifdef_directives.py b/scripts/maint/annotate_ifdef_directives.py index cd70b55c8c..9ca090d595 100755 --- a/scripts/maint/annotate_ifdef_directives.py +++ b/scripts/maint/annotate_ifdef_directives.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2017-2019, The Tor Project, Inc. # See LICENSE for licensing information diff --git a/scripts/maint/checkIncludes.py b/scripts/maint/checkIncludes.py index 2ca46347f0..ae0ccb9e12 100755 --- a/scripts/maint/checkIncludes.py +++ b/scripts/maint/checkIncludes.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright 2018 The Tor Project, Inc. See LICENSE file for licensing info. # This file is no longer here; see practracker/includes.py for this diff --git a/scripts/maint/checkManpageAlpha.py b/scripts/maint/checkManpageAlpha.py new file mode 100755 index 0000000000..70421c2fd1 --- /dev/null +++ b/scripts/maint/checkManpageAlpha.py @@ -0,0 +1,72 @@ +#!/usr/bin/python + +import difflib +import re +import sys + +# Assume we only use the "== Section Name" section title syntax +sectionheader_re = re.compile(r'^==+\s(.*)\s*$') + +# Assume we only use the "[[ItemName]]" anchor syntax +anchor_re = re.compile(r'^\[\[([^]]+)\]\]') + +class Reader(object): + def __init__(self): + self.d = {} + # Initial state is to gather section headers + self.getline = self._getsec + self.section = None + + def _getsec(self, line): + """Read a section header + + Prepare to gather anchors from subsequent lines. Don't change + state if the line isn't a section header. + """ + m = sectionheader_re.match(line) + if not m: + return + self.anchors = anchors = [] + self.d[m.group(1)] = anchors + self.getline = self._getanchor + + def _getanchor(self, line): + """Read an anchor for an item definition + + Append the anchor names to the list of items in the current + section. + """ + m = anchor_re.match(line) + if not m: + return self._getsec(line) + self.anchors.append(m.group(1)) + + def diffsort(self, key): + """Unified diff of unsorted and sorted item lists + """ + # Append newlines because difflib works better with them + a = [s + '\n' for s in self.d[key]] + b = sorted(a, key=str.lower) + return difflib.unified_diff(a, b, fromfile=key+' unsorted', + tofile=key+' sorted') + +def main(): + """Diff unsorted and sorted lists of option names in a manpage + + Use the file named by the first argument, or standard input if + there is none. + """ + try: + fname = sys.argv[1] + f = open(fname, 'r') + except IndexError: + f = sys.stdin + + reader = Reader() + for line in f: + reader.getline(line) + for key in sorted(reader.d.keys(), key=str.lower): + sys.stdout.writelines(reader.diffsort(key)) + +if __name__ == '__main__': + main() diff --git a/scripts/maint/checkShellScripts.sh b/scripts/maint/checkShellScripts.sh index 4c872c7ee0..0a423be29e 100755 --- a/scripts/maint/checkShellScripts.sh +++ b/scripts/maint/checkShellScripts.sh @@ -34,6 +34,9 @@ if [ ! -d "$TOPLEVEL/src" ]; then exit 1 fi +# Remove obsolete scripts generated from older versions of Tor +rm -f "$TOPLEVEL/contrib/dist/suse/tor.sh" "$TOPLEVEL/contrib/dist/tor.sh" + # Check *.sh scripts, but ignore the ones that we can't fix find "$TOPLEVEL/contrib" "$TOPLEVEL/doc" "$TOPLEVEL/scripts" "$TOPLEVEL/src" \ -name "*.sh" \ diff --git a/scripts/maint/checkSpace.pl b/scripts/maint/checkSpace.pl index f4e6f733c8..857ce6f6f1 100755 --- a/scripts/maint/checkSpace.pl +++ b/scripts/maint/checkSpace.pl @@ -23,6 +23,25 @@ if ($ARGV[0] =~ /^-/) { $C = ($lang eq '-C'); } +# hashmap of things where we allow spaces between them and (. +our %allow_space_after= map {$_, 1} qw{ + if while for switch return int unsigned elsif WINAPI + void __attribute__ op size_t double uint64_t + bool ssize_t + workqueue_reply_t hs_desc_decode_status_t + PRStatus + SMARTLIST_FOREACH_BEGIN SMARTLIST_FOREACH_END + HT_FOREACH + DIGESTMAP_FOREACH_MODIFY DIGESTMAP_FOREACH + DIGEST256MAP_FOREACH_MODIFY DIGEST256MAP_FOREACH + STRMAP_FOREACH_MODIFY STRMAP_FOREACH + SDMAP_FOREACH EIMAP_FOREACH RIMAP_FOREACH + MAP_FOREACH_MODIFY MAP_FOREACH + TOR_SIMPLEQ_FOREACH TOR_SIMPLEQ_FOREACH_SAFE + TOR_LIST_FOREACH TOR_LIST_FOREACH_SAFE + TOR_SLIST_FOREACH TOR_SLIST_FOREACH_SAFE +}; + our %basenames = (); our %guardnames = (); @@ -58,9 +77,9 @@ for my $fn (@ARGV) { } ## Warn about labels that don't have a space in front of them # (We indent every label at least one space) - if (/^[a-zA-Z_][a-zA-Z_0-9]*:/) { - msg "nosplabel:$fn:$.\n"; - } + #if (/^[a-zA-Z_][a-zA-Z_0-9]*:/) { + # msg "nosplabel:$fn:$.\n"; + #} ## Warn about trailing whitespace. # (We don't allow whitespace at the end of the line; make your # editor highlight it for you so you can stop adding it in.) @@ -111,7 +130,7 @@ for my $fn (@ARGV) { ## Terminals are still 80 columns wide in my world. I refuse to ## accept double-line lines. # (Don't make lines wider than 80 characters, including newline.) - if (/^.{80}/) { + if (/^.{80}/ and not /LCOV_EXCL/) { msg "Wide:$fn:$.\n"; } ### Juju to skip over comments and strings, since the tests @@ -128,12 +147,12 @@ for my $fn (@ARGV) { if ($isheader) { if ($seenguard == 0) { - if (/ifndef\s+(\S+)/) { + if (/^\s*\#\s*ifndef\s+(\S+)/) { ++$seenguard; $guardname = $1; } } elsif ($seenguard == 1) { - if (/^\#define (\S+)/) { + if (/^\s*\#\s*define (\S+)/) { ++$seenguard; if ($1 ne $guardname) { msg "GUARD:$fn:$.: Header guard macro mismatch.\n"; @@ -156,9 +175,8 @@ for my $fn (@ARGV) { # msg "//:$fn:$.\n"; s!//.*!!; } - ## Warn about unquoted braces preceded by non-space. - # (No character except a space should come before a {) - if (/([^\s'])\{/) { + ## Warn about unquoted braces preceded by unexpected character. + if (/([^\s'\)\(\{])\{/) { msg "$1\{:$fn:$.\n"; } ## Warn about double semi-colons at the end of a line. @@ -178,12 +196,7 @@ for my $fn (@ARGV) { # (Don't put a space between the name of a function and its # arguments.) if (/(\w+)\s\(([A-Z]*)/) { - if ($1 ne "if" and $1 ne "while" and $1 ne "for" and - $1 ne "switch" and $1 ne "return" and $1 ne "int" and - $1 ne "elsif" and $1 ne "WINAPI" and $2 ne "WINAPI" and - $1 ne "void" and $1 ne "__attribute__" and $1 ne "op" and - $1 ne "size_t" and $1 ne "double" and $1 ne "uint64_t" and - $1 ne "workqueue_reply_t" and $1 ne "bool") { + if (! $allow_space_after{$1} && $2 ne 'WINAPI') { msg "fn ():$fn:$.\n"; } } @@ -194,8 +207,8 @@ for my $fn (@ARGV) { if ($in_func_head || ($fn !~ /\.h$/ && /^[a-zA-Z0-9_]/ && ! /^(?:const |static )*(?:typedef|struct|union)[^\(]*$/ && - ! /= *\{$/ && ! /;$/)) { - if (/.\{$/){ + ! /= *\{$/ && ! /;$/) && ! /^[a-zA-Z0-9_]+\s*:/) { + if (/[^,\s]\s*\{$/){ msg "fn() {:$fn:$.\n"; $in_func_head = 0; } elsif (/^\S[^\(]* +\**[a-zA-Z0-9_]+\(/) { diff --git a/scripts/maint/checkspace_tests/expected.txt b/scripts/maint/checkspace_tests/expected.txt index 935b750ef9..38595ed373 100644 --- a/scripts/maint/checkspace_tests/expected.txt +++ b/scripts/maint/checkspace_tests/expected.txt @@ -5,7 +5,6 @@ tp fn():./dubious.c:15 Wide:./dubious.c:17 TAB:./dubious.c:24 - nosplabel:./dubious.c:26 CR:./dubious.c:30 Space@EOL:./dubious.c:32 non-K&R {:./dubious.c:39 diff --git a/scripts/maint/clang-format.sh b/scripts/maint/clang-format.sh new file mode 100755 index 0000000000..59832117b4 --- /dev/null +++ b/scripts/maint/clang-format.sh @@ -0,0 +1,41 @@ +#!/bin/sh +# Copyright 2020, The Tor Project, Inc. +# See LICENSE for licensing information. + +# +# DO NOT COMMIT OR MERGE CODE THAT IS RUN THROUGH THIS TOOL YET. +# +# WE ARE STILL DISCUSSING OUR DESIRED STYLE AND ITERATING ON IT. +# (12 Feb 2020) +# + +# This script runs "clang-format" and "codetool" in sequence over each of +# our source files, and replaces the original file only if it has changed. +# +# We can't just use clang-format -i, since we also want to use codetool to +# reformat a few things back to how we want them, and we want avoid changing +# the mtime on files that didn't actually change. + +set -e + +cd "$(dirname "$0")/../../src/" + +# Shellcheck complains that a for loop over find's output is unreliable, +# since there might be special characters in the output. But we happen +# to know that none of our C files have special characters or spaces in +# their names, so this is safe. +# +# shellcheck disable=SC2044 +for fname in $(find lib core feature app test tools -name '[^.]*.[ch]'); do + tmpfname="${fname}.clang_fmt.tmp" + rm -f "${tmpfname}" + clang-format --style=file "${fname}" > "${tmpfname}" + ../scripts/maint/codetool.py "${tmpfname}" + if cmp "${fname}" "${tmpfname}" >/dev/null 2>&1; then + echo "No change in ${fname}" + rm -f "${tmpfname}" + else + echo "Change in ${fname}" + mv "${tmpfname}" "${fname}" + fi +done diff --git a/scripts/maint/codetool.py b/scripts/maint/codetool.py new file mode 100755 index 0000000000..725712c0cc --- /dev/null +++ b/scripts/maint/codetool.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020, The Tor Project, Inc. +# See LICENSE for licensing information. + +# +# DO NOT COMMIT OR MERGE CODE THAT IS RUN THROUGH THIS TOOL YET. +# +# WE ARE STILL DISCUSSING OUR DESIRED STYLE AND ITERATING ON IT, +# ALONG WITH THE TOOLS THAT ACHIEVE IT. +# (12 Feb 2020) +# + +""" + This program uses a set of plugable filters to inspect and transform + our C code. +""" + +import os +import re +import sys + +class Filter: + """A Filter transforms a string containing a C program.""" + def __init__(self): + pass + + def transform(self, s): + return s + +class CompoundFilt(Filter): + """A CompoundFilt runs another set of filters, in sequence.""" + def __init__(self, items=()): + super().__init__() + self._filters = list(items) + + def add(self, filt): + self._filters.append(filt) + return self + + def transform(self, s): + for f in self._filters: + s = f.transform(s) + + return s + +class SplitError(Exception): + """Exception: raised if split_comments() can't understand a C file.""" + pass + +def split_comments(s): + r"""Iterate over the C code in 's', and yield a sequence of (code, + comment) pairs. Each pair will contain either a nonempty piece + of code, a nonempty comment, or both. + + >>> list(split_comments("hello // world\n")) + [('hello ', '// world'), ('\n', '')] + + >>> list(split_comments("a /* b cd */ efg // hi")) + [('a ', '/* b cd */'), (' efg ', '// hi')] + """ + + # Matches a block of code without any comments. + PAT_CODE = re.compile(r'''^(?: [^/"']+ | + "(?:[^\\"]+|\\.)*" | + '(?:[^\\']+|\\.)*' | + /[^/*] + )*''', re.VERBOSE|re.DOTALL) + + # Matches a C99 "//" comment. + PAT_C99_COMMENT = re.compile(r'^//.*$', re.MULTILINE) + + # Matches a C "/* */" comment. + PAT_C_COMMENT = re.compile(r'^/\*(?:[^*]|\*+[^*/])*\*+/', re.DOTALL) + + while True: + # Find some non-comment code at the start of the string. + m = PAT_CODE.match(s) + + # If we found some code here, save it and advance the string. + # Otherwise set 'code' to "". + if m: + code = m.group(0) + s = s[m.end():] + else: + code = "" + + # Now we have a comment, or the end of the string. Find out which + # one, and how long it is. + if s.startswith("//"): + m = PAT_C99_COMMENT.match(s) + else: + m = PAT_C_COMMENT.match(s) + + # If we got a comment, save it and advance the string. Otherwise + # set 'comment' to "". + if m: + comment = m.group(0) + s = s[m.end():] + else: + comment = "" + + # If we found no code and no comment, we should be at the end of + # the string... + if code == "" and comment == "": + if s: + # But in case we *aren't* at the end of the string, raise + # an error. + raise SplitError() + # ... all is well, we're done scanning the code. + return + + yield (code, comment) + +class IgnoreCommentsFilt(Filter): + """Wrapper: applies another filter to C code only, excluding comments. + """ + def __init__(self, filt): + super().__init__() + self._filt = filt + + def transform(self, s): + result = [] + for code, comment in split_comments(s): + result.append(self._filt.transform(code)) + result.append(comment) + return "".join(result) + + +class RegexFilt(Filter): + """A regex filter applies a regular expression to some C code.""" + def __init__(self, pat, replacement, flags=0): + super().__init__() + self._pat = re.compile(pat, flags) + self._replacement = replacement + + def transform(self, s): + s, _ = self._pat.subn(self._replacement, s) + return s + +def revise(fname, filt): + """Run 'filt' on the contents of the file in 'fname'. If any + changes are made, then replace the file with its new contents. + Otherwise, leave the file alone. + """ + contents = open(fname, 'r').read() + result = filt.transform(contents) + if result == contents: + return + + tmpname = "{}_codetool_tmp".format(fname) + try: + with open(tmpname, 'w') as f: + f.write(result) + os.rename(tmpname, fname) + except: + os.unlink(tmpname) + raise + +############################## +# Filtering rules. +############################## + +# Make sure that there is a newline after the first comma in a MOCK_IMPL() +BREAK_MOCK_IMPL = RegexFilt( + r'^MOCK_IMPL\(([^,]+),\s*(\S+)', + r'MOCK_IMPL(\1,\n\2', + re.MULTILINE) + +# Make sure there is no newline between } and a loop iteration terminator. +RESTORE_SMARTLIST_END = RegexFilt( + r'}\s*(SMARTLIST|DIGESTMAP|DIGEST256MAP|STRMAP|MAP)_FOREACH_END\s*\(', + r'} \1_FOREACH_END (', + re.MULTILINE) + +F = CompoundFilt() +F.add(IgnoreCommentsFilt(CompoundFilt([ + RESTORE_SMARTLIST_END, + BREAK_MOCK_IMPL]))) + +if __name__ == '__main__': + for fname in sys.argv[1:]: + revise(fname, F) diff --git a/scripts/maint/format_changelog.py b/scripts/maint/format_changelog.py index 32085c3602..93ab56e257 100755 --- a/scripts/maint/format_changelog.py +++ b/scripts/maint/format_changelog.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2014-2019, The Tor Project, Inc. # See LICENSE for licensing information # @@ -291,7 +291,7 @@ class ChangeLog(object): self.curgraf.append(line) else: - assert "This" is "unreachable" # noqa: F632 + assert False # This should be unreachable. def lint_head(self, line, head): m = re.match(r'^ *o ([^\(]+)((?:\([^\)]+\))?):', head) @@ -405,10 +405,31 @@ class ChangeLog(object): self.dumpEndOfSections() self.dumpEndOfChangelog() +# Map from issue prefix to pair of (visible prefix, url prefix) +ISSUE_PREFIX_MAP = { + "" : ( "", "tpo/core/tor" ), + "tor#" : ( "", "tpo/core/tor" ), + "chutney#" : ( "chutney#", "tpo/core/chutney" ), + "torspec#" : ( "torspec#", "tpo/core/torspec" ), + "trunnel#" : ( "trunnel#", "tpo/core/trunnel" ), + "torsocks#" : ( "torsocks#", "tpo/core/torsocks"), +} + # Let's turn bugs to html. -BUG_PAT = re.compile('(bug|ticket|issue|feature)\s+(\d{4,5})', re.I) +BUG_PAT = re.compile('(bug|ticket|issue|feature)\s+([\w/]+#)?(\d{4,6})', re.I) def bug_html(m): - return "%s <a href='https://bugs.torproject.org/%s'>%s</a>" % (m.group(1), m.group(2), m.group(2)) + kind = m.group(1) + prefix = m.group(2) or "" + bugno = m.group(3) + try: + disp_prefix, url_prefix = ISSUE_PREFIX_MAP[prefix] + except KeyError: + print("Can't figure out URL for {}{}".formt(prefix,bugno), + file=sys.stderr) + return "{} {}{}".format(kind, prefix, bugno) + + return "{} <a href='https://bugs.torproject.org/{}/{}'>{}{}</a>".format( + kind, url_prefix, bugno, disp_prefix, bugno) class HTMLChangeLog(ChangeLog): def __init__(self, *args, **kwargs): diff --git a/scripts/maint/lintChanges.py b/scripts/maint/lintChanges.py index 88a865a572..cf7b09fcc3 100755 --- a/scripts/maint/lintChanges.py +++ b/scripts/maint/lintChanges.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Future imports for Python 2.7, mandatory in 3.0 from __future__ import division @@ -25,7 +25,12 @@ KNOWN_GROUPS = set([ "Code simplification and refactoring", "Removed features", "Deprecated features", - "Directory authority changes"]) + "Directory authority changes", + + # These aren't preferred, but sortChanges knows how to clean them up. + "Code simplifications and refactoring", + "Code simplification and refactorings", + "Code simplifications and refactorings"]) NEEDS_SUBCATEGORIES = set([ "Minor bugfix", diff --git a/scripts/maint/locatemissingdoxygen.py b/scripts/maint/locatemissingdoxygen.py index 7733977359..a2844346d6 100755 --- a/scripts/maint/locatemissingdoxygen.py +++ b/scripts/maint/locatemissingdoxygen.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python """ This script parses the stderr output of doxygen and looks for undocumented diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index d89b80c1b3..35e860d8b1 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -33,23 +33,22 @@ # # Remember: It is better to fix the problem than to add a new exception! -problem file-size /src/app/config/config.c 7400 +problem file-size /src/app/config/config.c 7525 problem include-count /src/app/config/config.c 80 -problem function-size /src/app/config/config.c:options_act_reversible() 298 problem function-size /src/app/config/config.c:options_act() 381 -problem function-size /src/app/config/config.c:resolve_my_address() 190 -problem function-size /src/app/config/config.c:options_validate_cb() 780 -problem function-size /src/app/config/config.c:options_init_from_torrc() 188 +problem function-size /src/app/config/config.c:options_validate_cb() 794 +problem function-size /src/app/config/config.c:options_init_from_torrc() 192 problem function-size /src/app/config/config.c:options_init_from_string() 103 problem function-size /src/app/config/config.c:options_init_logs() 125 problem function-size /src/app/config/config.c:parse_bridge_line() 104 -problem function-size /src/app/config/config.c:pt_parse_transport_line() 189 +problem function-size /src/app/config/config.c:pt_parse_transport_line() 190 problem function-size /src/app/config/config.c:parse_dir_authority_line() 150 problem function-size /src/app/config/config.c:parse_dir_fallback_line() 101 -problem function-size /src/app/config/config.c:port_parse_config() 450 +problem function-size /src/app/config/config.c:port_parse_config() 435 problem function-size /src/app/config/config.c:parse_ports() 132 -problem file-size /src/app/config/or_options_st.h 1115 -problem include-count /src/app/main/main.c 69 +problem function-size /src/app/config/resolve_addr.c:resolve_my_address() 191 +problem file-size /src/app/config/or_options_st.h 1050 +problem include-count /src/app/main/main.c 68 problem function-size /src/app/main/main.c:dumpstats() 102 problem function-size /src/app/main/main.c:tor_init() 101 problem function-size /src/app/main/main.c:sandbox_init_filter() 291 @@ -61,22 +60,21 @@ problem dependency-violation /src/core/crypto/onion_crypto.c 5 problem dependency-violation /src/core/crypto/onion_fast.c 1 problem dependency-violation /src/core/crypto/onion_tap.c 3 problem dependency-violation /src/core/crypto/relay_crypto.c 9 -problem file-size /src/core/mainloop/connection.c 5577 -problem include-count /src/core/mainloop/connection.c 62 -problem function-size /src/core/mainloop/connection.c:connection_free_minimal() 185 -problem function-size /src/core/mainloop/connection.c:connection_listener_new() 324 +problem file-size /src/core/mainloop/connection.c 5700 +problem include-count /src/core/mainloop/connection.c 65 +problem function-size /src/core/mainloop/connection.c:connection_free_minimal() 181 +problem function-size /src/core/mainloop/connection.c:connection_listener_new() 325 problem function-size /src/core/mainloop/connection.c:connection_handle_listener_read() 161 -problem function-size /src/core/mainloop/connection.c:connection_proxy_connect() 148 problem function-size /src/core/mainloop/connection.c:connection_read_proxy_handshake() 153 problem function-size /src/core/mainloop/connection.c:retry_listener_ports() 112 problem function-size /src/core/mainloop/connection.c:connection_handle_read_impl() 111 -problem function-size /src/core/mainloop/connection.c:connection_buf_read_from_socket() 180 +problem function-size /src/core/mainloop/connection.c:connection_buf_read_from_socket() 186 problem function-size /src/core/mainloop/connection.c:connection_handle_write_impl() 241 problem function-size /src/core/mainloop/connection.c:assert_connection_ok() 143 problem dependency-violation /src/core/mainloop/connection.c 47 problem dependency-violation /src/core/mainloop/cpuworker.c 12 problem include-count /src/core/mainloop/mainloop.c 64 -problem function-size /src/core/mainloop/mainloop.c:conn_close_if_marked() 108 +problem function-size /src/core/mainloop/mainloop.c:conn_close_if_marked() 107 problem function-size /src/core/mainloop/mainloop.c:run_connection_housekeeping() 123 problem dependency-violation /src/core/mainloop/mainloop.c 50 problem dependency-violation /src/core/mainloop/mainloop_pubsub.c 1 @@ -85,9 +83,9 @@ problem dependency-violation /src/core/mainloop/netstatus.c 4 problem dependency-violation /src/core/mainloop/periodic.c 2 problem dependency-violation /src/core/or/address_set.c 1 problem dependency-violation /src/core/or/cell_queue_st.h 1 -problem file-size /src/core/or/channel.c 3487 +problem file-size /src/core/or/channel.c 3500 problem dependency-violation /src/core/or/channel.c 9 -problem file-size /src/core/or/channel.h 781 +problem file-size /src/core/or/channel.h 800 problem dependency-violation /src/core/or/channel.h 1 problem dependency-violation /src/core/or/channelpadding.c 6 problem function-size /src/core/or/channeltls.c:channel_tls_handle_var_cell() 160 @@ -95,24 +93,22 @@ problem function-size /src/core/or/channeltls.c:channel_tls_process_versions_cel problem function-size /src/core/or/channeltls.c:channel_tls_process_netinfo_cell() 214 problem function-size /src/core/or/channeltls.c:channel_tls_process_certs_cell() 246 problem function-size /src/core/or/channeltls.c:channel_tls_process_authenticate_cell() 202 -problem dependency-violation /src/core/or/channeltls.c 10 -problem include-count /src/core/or/circuitbuild.c 54 +problem dependency-violation /src/core/or/channeltls.c 11 +problem include-count /src/core/or/circuitbuild.c 53 problem function-size /src/core/or/circuitbuild.c:get_unique_circ_id_by_chan() 128 -problem function-size /src/core/or/circuitbuild.c:circuit_extend() 147 problem function-size /src/core/or/circuitbuild.c:choose_good_exit_server_general() 206 problem dependency-violation /src/core/or/circuitbuild.c 25 problem include-count /src/core/or/circuitlist.c 55 problem function-size /src/core/or/circuitlist.c:HT_PROTOTYPE() 109 problem function-size /src/core/or/circuitlist.c:circuit_free_() 146 problem function-size /src/core/or/circuitlist.c:circuit_find_to_cannibalize() 101 -problem function-size /src/core/or/circuitlist.c:circuit_about_to_free() 120 problem function-size /src/core/or/circuitlist.c:circuits_handle_oom() 117 problem dependency-violation /src/core/or/circuitlist.c 19 problem dependency-violation /src/core/or/circuitlist.h 1 problem function-size /src/core/or/circuitmux.c:circuitmux_set_policy() 109 problem function-size /src/core/or/circuitmux.c:circuitmux_attach_circuit() 113 problem dependency-violation /src/core/or/circuitmux_ewma.c 2 -problem file-size /src/core/or/circuitpadding.c 3098 +problem file-size /src/core/or/circuitpadding.c 3101 problem function-size /src/core/or/circuitpadding.c:circpad_machine_schedule_padding() 113 problem dependency-violation /src/core/or/circuitpadding.c 6 problem file-size /src/core/or/circuitpadding.h 813 @@ -121,9 +117,9 @@ problem function-size /src/core/or/circuitpadding_machines.c:circpad_machine_cli problem dependency-violation /src/core/or/circuitpadding_machines.c 1 problem function-size /src/core/or/circuitstats.c:circuit_build_times_parse_state() 123 problem dependency-violation /src/core/or/circuitstats.c 11 -problem file-size /src/core/or/circuituse.c 3162 +problem file-size /src/core/or/circuituse.c 3195 problem function-size /src/core/or/circuituse.c:circuit_is_acceptable() 128 -problem function-size /src/core/or/circuituse.c:circuit_expire_building() 394 +problem function-size /src/core/or/circuituse.c:circuit_expire_building() 389 problem function-size /src/core/or/circuituse.c:circuit_log_ancient_one_hop_circuits() 126 problem function-size /src/core/or/circuituse.c:circuit_build_failed() 149 problem function-size /src/core/or/circuituse.c:circuit_launch_by_extend_info() 108 @@ -132,8 +128,8 @@ problem function-size /src/core/or/circuituse.c:connection_ap_handshake_attach_c problem dependency-violation /src/core/or/circuituse.c 24 problem function-size /src/core/or/command.c:command_process_create_cell() 156 problem function-size /src/core/or/command.c:command_process_relay_cell() 132 -problem dependency-violation /src/core/or/command.c 8 -problem file-size /src/core/or/connection_edge.c 4640 +problem dependency-violation /src/core/or/command.c 9 +problem file-size /src/core/or/connection_edge.c 4655 problem include-count /src/core/or/connection_edge.c 65 problem function-size /src/core/or/connection_edge.c:connection_ap_expire_beginning() 117 problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_rewrite() 193 @@ -145,24 +141,21 @@ problem function-size /src/core/or/connection_edge.c:connection_exit_begin_conn( problem function-size /src/core/or/connection_edge.c:connection_exit_connect() 102 problem dependency-violation /src/core/or/connection_edge.c 27 problem dependency-violation /src/core/or/connection_edge.h 1 -problem file-size /src/core/or/connection_or.c 3122 -problem include-count /src/core/or/connection_or.c 51 problem function-size /src/core/or/connection_or.c:connection_or_group_set_badness_() 105 problem function-size /src/core/or/connection_or.c:connection_or_client_learned_peer_id() 142 -problem function-size /src/core/or/connection_or.c:connection_or_compute_authenticate_cell_body() 231 -problem dependency-violation /src/core/or/connection_or.c 20 +problem dependency-violation /src/core/or/connection_or.c 21 problem dependency-violation /src/core/or/dos.c 6 problem dependency-violation /src/core/or/onion.c 2 -problem file-size /src/core/or/or.h 1107 -problem include-count /src/core/or/or.h 49 +problem file-size /src/core/or/or.h 1105 +problem include-count /src/core/or/or.h 48 problem dependency-violation /src/core/or/or.h 1 problem dependency-violation /src/core/or/or_periodic.c 1 -problem file-size /src/core/or/policies.c 3249 +problem file-size /src/core/or/policies.c 3182 problem function-size /src/core/or/policies.c:policy_summarize() 107 problem dependency-violation /src/core/or/policies.c 14 problem function-size /src/core/or/protover.c:protover_all_supported() 117 problem dependency-violation /src/core/or/reasons.c 2 -problem file-size /src/core/or/relay.c 3264 +problem file-size /src/core/or/relay.c 3300 problem function-size /src/core/or/relay.c:circuit_receive_relay_cell() 127 problem function-size /src/core/or/relay.c:relay_send_command_from_edge_() 109 problem function-size /src/core/or/relay.c:connection_ap_process_end_not_open() 192 @@ -170,14 +163,14 @@ problem function-size /src/core/or/relay.c:connection_edge_process_relay_cell_no problem function-size /src/core/or/relay.c:handle_relay_cell_command() 369 problem function-size /src/core/or/relay.c:connection_edge_package_raw_inbuf() 128 problem function-size /src/core/or/relay.c:circuit_resume_edge_reading_helper() 146 -problem dependency-violation /src/core/or/relay.c 16 +problem dependency-violation /src/core/or/relay.c 17 problem dependency-violation /src/core/or/scheduler.c 1 problem function-size /src/core/or/scheduler_kist.c:kist_scheduler_run() 171 problem dependency-violation /src/core/or/scheduler_kist.c 2 problem function-size /src/core/or/scheduler_vanilla.c:vanilla_scheduler_run() 109 problem dependency-violation /src/core/or/scheduler_vanilla.c 1 problem dependency-violation /src/core/or/sendme.c 2 -problem dependency-violation /src/core/or/status.c 12 +problem dependency-violation /src/core/or/status.c 13 problem function-size /src/core/or/versions.c:tor_version_parse() 104 problem dependency-violation /src/core/proto/proto_cell.c 3 problem dependency-violation /src/core/proto/proto_control0.c 1 @@ -186,53 +179,53 @@ problem dependency-violation /src/core/proto/proto_http.c 1 problem function-size /src/core/proto/proto_socks.c:parse_socks_client() 110 problem dependency-violation /src/core/proto/proto_socks.c 8 problem function-size /src/feature/client/addressmap.c:addressmap_rewrite() 109 -problem function-size /src/feature/client/bridges.c:rewrite_node_address_for_bridge() 126 +problem function-size /src/feature/client/bridges.c:rewrite_node_address_for_bridge() 125 problem function-size /src/feature/client/circpathbias.c:pathbias_measure_close_rate() 108 problem function-size /src/feature/client/dnsserv.c:evdns_server_callback() 153 -problem file-size /src/feature/client/entrynodes.c 3825 +problem file-size /src/feature/client/entrynodes.c 4000 problem function-size /src/feature/client/entrynodes.c:entry_guards_upgrade_waiting_circuits() 155 problem function-size /src/feature/client/entrynodes.c:entry_guard_parse_from_state() 246 -problem file-size /src/feature/client/entrynodes.h 639 +problem file-size /src/feature/client/entrynodes.h 700 problem function-size /src/feature/client/transports.c:handle_proxy_line() 108 problem function-size /src/feature/client/transports.c:parse_method_line_helper() 110 -problem function-size /src/feature/client/transports.c:create_managed_proxy_environment() 109 +problem function-size /src/feature/client/transports.c:create_managed_proxy_environment() 111 problem function-size /src/feature/control/control.c:connection_control_process_inbuf() 113 problem function-size /src/feature/control/control_auth.c:handle_control_authenticate() 186 problem function-size /src/feature/control/control_cmd.c:handle_control_extendcircuit() 150 problem function-size /src/feature/control/control_cmd.c:handle_control_add_onion() 256 problem function-size /src/feature/control/control_cmd.c:add_onion_helper_keyarg() 118 problem function-size /src/feature/control/control_events.c:control_event_stream_status() 124 -problem include-count /src/feature/control/control_getinfo.c 54 +problem include-count /src/feature/control/control_getinfo.c 56 problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_misc() 108 problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_dir() 297 problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_events() 234 problem function-size /src/feature/dirauth/bwauth.c:dirserv_read_measured_bandwidths() 121 -problem file-size /src/feature/dirauth/dirvote.c 4700 -problem include-count /src/feature/dirauth/dirvote.c 53 -problem function-size /src/feature/dirauth/dirvote.c:format_networkstatus_vote() 231 +problem file-size /src/feature/dirauth/dirvote.c 4734 +problem include-count /src/feature/dirauth/dirvote.c 55 +problem function-size /src/feature/dirauth/dirvote.c:format_networkstatus_vote() 230 problem function-size /src/feature/dirauth/dirvote.c:networkstatus_compute_bw_weights_v10() 233 -problem function-size /src/feature/dirauth/dirvote.c:networkstatus_compute_consensus() 956 +problem function-size /src/feature/dirauth/dirvote.c:networkstatus_compute_consensus() 952 problem function-size /src/feature/dirauth/dirvote.c:networkstatus_add_detached_signatures() 119 -problem function-size /src/feature/dirauth/dirvote.c:dirvote_add_vote() 162 +problem function-size /src/feature/dirauth/dirvote.c:dirvote_add_vote() 161 problem function-size /src/feature/dirauth/dirvote.c:dirvote_compute_consensuses() 164 -problem function-size /src/feature/dirauth/dirvote.c:dirserv_generate_networkstatus_vote_obj() 283 +problem function-size /src/feature/dirauth/dirvote.c:dirserv_generate_networkstatus_vote_obj() 281 problem function-size /src/feature/dirauth/dsigs_parse.c:networkstatus_parse_detached_signatures() 196 problem function-size /src/feature/dirauth/guardfraction.c:dirserv_read_guardfraction_file_from_str() 109 problem function-size /src/feature/dirauth/process_descs.c:dirserv_add_descriptor() 125 problem function-size /src/feature/dirauth/shared_random.c:should_keep_commit() 109 -problem function-size /src/feature/dirauth/voteflags.c:dirserv_compute_performance_thresholds() 172 +problem function-size /src/feature/dirauth/voteflags.c:dirserv_compute_performance_thresholds() 175 problem function-size /src/feature/dircache/consdiffmgr.c:consdiffmgr_cleanup() 115 problem function-size /src/feature/dircache/consdiffmgr.c:consdiffmgr_rescan_flavor_() 111 problem function-size /src/feature/dircache/consdiffmgr.c:consensus_diff_worker_threadfn() 132 problem function-size /src/feature/dircache/dircache.c:handle_get_current_consensus() 165 problem function-size /src/feature/dircache/dircache.c:directory_handle_command_post() 124 -problem file-size /src/feature/dirclient/dirclient.c 3165 -problem include-count /src/feature/dirclient/dirclient.c 51 +problem file-size /src/feature/dirclient/dirclient.c 3204 +problem include-count /src/feature/dirclient/dirclient.c 54 problem function-size /src/feature/dirclient/dirclient.c:directory_get_from_dirserver() 126 problem function-size /src/feature/dirclient/dirclient.c:directory_initiate_request() 201 problem function-size /src/feature/dirclient/dirclient.c:directory_send_command() 239 problem function-size /src/feature/dirclient/dirclient.c:dir_client_decompress_response_body() 111 -problem function-size /src/feature/dirclient/dirclient.c:connection_dir_client_reached_eof() 189 +problem function-size /src/feature/dirclient/dirclient.c:connection_dir_client_reached_eof() 199 problem function-size /src/feature/dirclient/dirclient.c:handle_response_fetch_consensus() 104 problem function-size /src/feature/dircommon/consdiff.c:gen_ed_diff() 203 problem function-size /src/feature/dircommon/consdiff.c:apply_ed_diff() 158 @@ -241,50 +234,47 @@ problem function-size /src/feature/dirparse/ns_parse.c:routerstatus_parse_entry_ problem function-size /src/feature/dirparse/ns_parse.c:networkstatus_verify_bw_weights() 389 problem function-size /src/feature/dirparse/ns_parse.c:networkstatus_parse_vote_from_string() 635 problem function-size /src/feature/dirparse/parsecommon.c:tokenize_string() 101 -problem function-size /src/feature/dirparse/parsecommon.c:get_next_token() 158 +problem function-size /src/feature/dirparse/parsecommon.c:get_next_token() 165 problem function-size /src/feature/dirparse/routerparse.c:router_parse_entry_from_string() 554 problem function-size /src/feature/dirparse/routerparse.c:extrainfo_parse_entry_from_string() 208 problem function-size /src/feature/hibernate/hibernate.c:accounting_parse_options() 109 problem function-size /src/feature/hs/hs_cell.c:hs_cell_build_establish_intro() 115 -problem function-size /src/feature/hs/hs_cell.c:hs_cell_parse_introduce2() 152 -problem function-size /src/feature/hs/hs_client.c:send_introduce1() 103 -problem function-size /src/feature/hs/hs_client.c:hs_config_client_authorization() 107 +problem function-size /src/feature/hs/hs_cell.c:hs_cell_parse_introduce2() 134 +problem function-size /src/feature/hs/hs_client.c:send_introduce1() 108 problem function-size /src/feature/hs/hs_common.c:hs_get_responsible_hsdirs() 102 -problem function-size /src/feature/hs/hs_config.c:config_service_v3() 107 -problem function-size /src/feature/hs/hs_config.c:config_generic_service() 138 -problem function-size /src/feature/hs/hs_descriptor.c:desc_encode_v3() 101 problem function-size /src/feature/hs/hs_descriptor.c:decrypt_desc_layer() 111 problem function-size /src/feature/hs/hs_descriptor.c:decode_introduction_point() 122 problem function-size /src/feature/hs/hs_descriptor.c:desc_decode_superencrypted_v3() 107 problem function-size /src/feature/hs/hs_descriptor.c:desc_decode_encrypted_v3() 109 -problem file-size /src/feature/hs/hs_service.c 4172 +problem file-size /src/feature/hs/hs_service.c 4300 problem function-size /src/feature/keymgt/loadkey.c:ed_key_init_from_file() 326 problem function-size /src/feature/nodelist/authcert.c:trusted_dirs_load_certs_from_string() 123 problem function-size /src/feature/nodelist/authcert.c:authority_certs_fetch_missing() 295 -problem function-size /src/feature/nodelist/fmt_routerstatus.c:routerstatus_format_entry() 162 +problem function-size /src/feature/nodelist/fmt_routerstatus.c:routerstatus_format_entry() 158 problem function-size /src/feature/nodelist/microdesc.c:microdesc_cache_rebuild() 134 -problem include-count /src/feature/nodelist/networkstatus.c 63 +problem include-count /src/feature/nodelist/networkstatus.c 65 problem function-size /src/feature/nodelist/networkstatus.c:networkstatus_check_consensus_signature() 175 problem function-size /src/feature/nodelist/networkstatus.c:networkstatus_set_current_consensus() 289 problem function-size /src/feature/nodelist/node_select.c:router_pick_directory_server_impl() 122 -problem function-size /src/feature/nodelist/node_select.c:compute_weighted_bandwidths() 203 +problem function-size /src/feature/nodelist/node_select.c:compute_weighted_bandwidths() 204 problem function-size /src/feature/nodelist/node_select.c:router_pick_trusteddirserver_impl() 112 problem function-size /src/feature/nodelist/nodelist.c:compute_frac_paths_available() 190 -problem file-size /src/feature/nodelist/routerlist.c 3239 +problem file-size /src/feature/nodelist/routerlist.c 3247 problem function-size /src/feature/nodelist/routerlist.c:router_rebuild_store() 148 problem function-size /src/feature/nodelist/routerlist.c:router_add_to_routerlist() 168 problem function-size /src/feature/nodelist/routerlist.c:routerlist_remove_old_routers() 121 -problem function-size /src/feature/nodelist/routerlist.c:update_consensus_router_descriptor_downloads() 135 +problem function-size /src/feature/nodelist/routerlist.c:update_consensus_router_descriptor_downloads() 142 problem function-size /src/feature/nodelist/routerlist.c:update_extrainfo_downloads() 103 problem function-size /src/feature/relay/dns.c:dns_resolve_impl() 131 problem function-size /src/feature/relay/dns.c:configure_nameservers() 161 problem function-size /src/feature/relay/dns.c:evdns_callback() 108 -problem file-size /src/feature/relay/router.c 3520 +problem function-size /src/feature/relay/relay_handshake.c:connection_or_compute_authenticate_cell_body() 231 +problem file-size /src/feature/relay/router.c 3600 problem include-count /src/feature/relay/router.c 57 -problem function-size /src/feature/relay/router.c:init_keys() 252 +problem function-size /src/feature/relay/router.c:init_keys() 254 problem function-size /src/feature/relay/router.c:get_my_declared_family() 114 -problem function-size /src/feature/relay/router.c:router_build_fresh_unsigned_routerinfo() 136 -problem function-size /src/feature/relay/router.c:router_dump_router_to_string() 367 +problem function-size /src/feature/relay/router.c:router_build_fresh_unsigned_routerinfo() 113 +problem function-size /src/feature/relay/router.c:router_dump_router_to_string() 372 problem function-size /src/feature/relay/routerkeys.c:load_ed_keys() 294 problem function-size /src/feature/rend/rendcache.c:rend_cache_store_v2_desc_as_client() 190 problem function-size /src/feature/rend/rendclient.c:rend_client_send_introduction() 219 @@ -292,9 +282,9 @@ problem function-size /src/feature/rend/rendcommon.c:rend_encode_v2_descriptors( problem function-size /src/feature/rend/rendmid.c:rend_mid_establish_intro_legacy() 105 problem function-size /src/feature/rend/rendparse.c:rend_parse_v2_service_descriptor() 181 problem function-size /src/feature/rend/rendparse.c:rend_parse_introduction_points() 129 -problem file-size /src/feature/rend/rendservice.c 4522 +problem file-size /src/feature/rend/rendservice.c 4504 problem function-size /src/feature/rend/rendservice.c:rend_service_prune_list_impl_() 107 -problem function-size /src/feature/rend/rendservice.c:rend_config_service() 162 +problem function-size /src/feature/rend/rendservice.c:rend_config_service() 143 problem function-size /src/feature/rend/rendservice.c:rend_service_load_auth_keys() 178 problem function-size /src/feature/rend/rendservice.c:rend_service_receive_introduction() 334 problem function-size /src/feature/rend/rendservice.c:rend_service_parse_intro_for_v3() 111 @@ -315,11 +305,10 @@ problem function-size /src/lib/encoding/confline.c:parse_config_line_from_str_ve problem function-size /src/lib/encoding/cstring.c:unescape_string() 108 problem function-size /src/lib/fs/dir.c:check_private_dir() 230 problem function-size /src/lib/math/prob_distr.c:sample_uniform_interval() 145 -problem function-size /src/lib/net/address.c:tor_addr_parse_mask_ports() 194 +problem function-size /src/lib/net/address.c:tor_addr_parse_mask_ports() 195 problem function-size /src/lib/net/address.c:tor_addr_compare_masked() 110 problem function-size /src/lib/net/inaddr.c:tor_inet_pton() 107 problem function-size /src/lib/net/socketpair.c:tor_ersatz_socketpair() 102 -problem function-size /src/lib/osinfo/uname.c:get_uname() 116 problem function-size /src/lib/process/process_unix.c:process_unix_exec() 213 problem function-size /src/lib/process/process_win32.c:process_win32_exec() 151 problem function-size /src/lib/process/process_win32.c:process_win32_create_pipe() 109 diff --git a/scripts/maint/practracker/includes.py b/scripts/maint/practracker/includes.py index fe0f32e253..a5ee728824 100755 --- a/scripts/maint/practracker/includes.py +++ b/scripts/maint/practracker/includes.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright 2018 The Tor Project, Inc. See LICENSE file for licensing info. """This script looks through all the directories for files matching *.c or @@ -59,6 +59,8 @@ ALLOWED_PATTERNS = [ re.compile(r'^micro-revision.i$'), ] +TOPDIR = "src" + def pattern_is_normal(s): for p in ALLOWED_PATTERNS: if p.match(s): @@ -136,6 +138,29 @@ class Rules(object): return allowed + +def normalize_srcdir(fname): + """given the name of a source directory or file, return its name + relative to `src` in a unix-like format. + """ + orig = fname + dirname, dirfile = os.path.split(fname) + if re.match(r'.*\.[ch]$', dirfile): + fname = dirname + + # Now we have a directory. + dirname, result = os.path.split(fname) + for _ in range(100): + # prevent excess looping in case I missed a tricky case + dirname, dirpart = os.path.split(dirname) + if dirpart == 'src' or dirname == "": + #print(orig,"=>",result) + return result + result = "{}/{}".format(dirpart,result) + + print("No progress!") + assert False + include_rules_cache = {} def load_include_rules(fname): @@ -173,6 +198,27 @@ def remove_self_edges(graph): for k in list(graph): graph[k] = [ d for d in graph[k] if d != k ] +def closure(graph): + """Takes a directed graph in as an adjacency mapping (a mapping from + node to a list of the nodes to which it connects), and completes + its closure. + """ + graph = graph.copy() + changed = False + for k in graph.keys(): + graph[k] = set(graph[k]) + while True: + for k in graph.keys(): + sz = len(graph[k]) + for v in list(graph[k]): + graph[k].update(graph.get(v, [])) + if sz != len(graph[k]): + changed = True + + if not changed: + return graph + changed = False + def toposort(graph, limit=100): """Takes a directed graph in as an adjacency mapping (a mapping from node to a list of the nodes to which it connects). Tries to @@ -233,8 +279,38 @@ def walk_c_files(topdir="src"): for err in consider_include_rules(fullpath, f): yield err +def open_or_stdin(fname): + if fname == '-': + return sys.stdin + else: + return open(fname) + +def check_subsys_file(fname, uses_dirs): + if not uses_dirs: + # We're doing a distcheck build, or for some other reason there are + # no .may_include files. + print("SKIPPING") + return False + + uses_dirs = { normalize_srcdir(k) : { normalize_srcdir(d) for d in v } + for (k,v) in uses_dirs.items() } + uses_closure = closure(uses_dirs) + ok = True + previous_subsystems = [] + + with open_or_stdin(fname) as f: + for line in f: + _, name, fname = line.split() + fname = normalize_srcdir(fname) + for prev in previous_subsystems: + if fname in uses_closure[prev]: + print("INVERSION: {} uses {}".format(prev,fname)) + ok = False + previous_subsystems.append(fname) + return not ok + def run_check_includes(topdir, list_unused=False, log_sorted_levels=False, - list_advisories=False): + list_advisories=False, check_subsystem_order=None): trouble = False for err in walk_c_files(topdir): @@ -259,6 +335,11 @@ def run_check_includes(topdir, list_unused=False, log_sorted_levels=False, uses_dirs[rules.incpath] = rules.getAllowedDirectories() remove_self_edges(uses_dirs) + + if check_subsystem_order: + if check_subsys_file(check_subsystem_order, uses_dirs): + sys.exit(1) + all_levels = toposort(uses_dirs) if log_sorted_levels: @@ -282,14 +363,19 @@ def main(argv): help="List unused lines in .may_include files.") parser.add_argument("--list-advisories", action="store_true", help="List advisories as well as forbidden includes") + parser.add_argument("--check-subsystem-order", action="store", + help="Check a list of subsystems for ordering") parser.add_argument("topdir", default="src", nargs="?", help="Top-level directory for the tor source") args = parser.parse_args(argv[1:]) + global TOPDIR + TOPDIR = args.topdir run_check_includes(topdir=args.topdir, log_sorted_levels=args.toposort, list_unused=args.list_unused, - list_advisories=args.list_advisories) + list_advisories=args.list_advisories, + check_subsystem_order=args.check_subsystem_order) if __name__ == '__main__': main(sys.argv) diff --git a/scripts/maint/practracker/metrics.py b/scripts/maint/practracker/metrics.py index ae88b84f31..300a4501a9 100644 --- a/scripts/maint/practracker/metrics.py +++ b/scripts/maint/practracker/metrics.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Implementation of various source code metrics. # These are currently ad-hoc string operations and regexps. diff --git a/scripts/maint/practracker/practracker.py b/scripts/maint/practracker/practracker.py index 79b13cb056..76ffd64cfb 100755 --- a/scripts/maint/practracker/practracker.py +++ b/scripts/maint/practracker/practracker.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python """ Best-practices tracker for Tor source code. diff --git a/scripts/maint/practracker/practracker_tests.py b/scripts/maint/practracker/practracker_tests.py index 8d0418880c..e03c9e05ae 100755 --- a/scripts/maint/practracker/practracker_tests.py +++ b/scripts/maint/practracker/practracker_tests.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python """Some simple tests for practracker metrics""" diff --git a/scripts/maint/rectify_include_paths.py b/scripts/maint/rectify_include_paths.py index 111cf816ce..6c7b252535 100755 --- a/scripts/maint/rectify_include_paths.py +++ b/scripts/maint/rectify_include_paths.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Future imports for Python 2.7, mandatory in 3.0 from __future__ import division @@ -29,6 +29,12 @@ def get_include_map(): exclude(["ext", "win32"], dirnames) for fname in fnames: + # Avoid editor temporary files + if fname.startswith("."): + continue + if fname.startswith("#"): + continue + if fname.endswith(".h"): if fname in includes: warn("Multiple headers named %s"%fname) @@ -63,6 +69,12 @@ for dirpath,dirnames,fnames in os.walk("src"): exclude(["trunnel"], dirnames) for fname in fnames: + # Avoid editor temporary files + if fname.startswith("."): + continue + if fname.startswith("#"): + continue + if fname.endswith(".c") or fname.endswith(".h"): fname = os.path.join(dirpath, fname) tmpfile = fname+".tmp" diff --git a/scripts/maint/redox.py b/scripts/maint/redox.py index 3ad3e3f1b8..12b02c8a44 100755 --- a/scripts/maint/redox.py +++ b/scripts/maint/redox.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # # Copyright (c) 2008-2019, The Tor Project, Inc. # See LICENSE for licensing information. diff --git a/scripts/maint/rename_c_identifier.py b/scripts/maint/rename_c_identifier.py index 6e0c1d8cf1..77802e10f3 100755 --- a/scripts/maint/rename_c_identifier.py +++ b/scripts/maint/rename_c_identifier.py @@ -44,7 +44,8 @@ def is_c_file(fn): False """ fn = os.path.split(fn)[1] - if fn.startswith("."): + # Avoid editor temporary files + if fn.startswith(".") or fn.startswith("#"): return False ext = os.path.splitext(fn)[1] return ext in {".c", ".h", ".i", ".inc"} diff --git a/scripts/maint/run_check_subsystem_order.sh b/scripts/maint/run_check_subsystem_order.sh new file mode 100755 index 0000000000..4ec73bfd56 --- /dev/null +++ b/scripts/maint/run_check_subsystem_order.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -e + +TOR="${abs_top_builddir:-.}/src/app/tor" + +INCLUDES_PY="${abs_top_srcdir:-.}/scripts/maint/practracker/includes.py" + +if ! test -x "${INCLUDES_PY}" ; then + echo "skip" + exit 77 +fi + +"${TOR}" --dbg-dump-subsystem-list | \ + "${INCLUDES_PY}" --check-subsystem-order - "${abs_top_srcdir}/src" + +echo ok diff --git a/scripts/maint/sortChanges.py b/scripts/maint/sortChanges.py index 2e049b1e53..5f6324e387 100755 --- a/scripts/maint/sortChanges.py +++ b/scripts/maint/sortChanges.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Copyright (c) 2014-2019, The Tor Project, Inc. # See LICENSE for licensing information |