diff options
Diffstat (limited to 'scripts')
52 files changed, 2092 insertions, 396 deletions
diff --git a/scripts/coccinelle/apply.sh b/scripts/coccinelle/apply.sh new file mode 100755 index 0000000000..f531d7fa32 --- /dev/null +++ b/scripts/coccinelle/apply.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +# apply.sh: +# run spatch with appropriate includes and builtins for the Tor source code + +top="$(dirname "$0")/../.." + +spatch -macro_file_builtins "$top"/scripts/coccinelle/tor-coccinelle.h \ + -I "$top" -I "$top"/src -I "$top"/ext --defined COCCI "$@" diff --git a/scripts/coccinelle/check_cocci_parse.sh b/scripts/coccinelle/check_cocci_parse.sh new file mode 100755 index 0000000000..aaa586c093 --- /dev/null +++ b/scripts/coccinelle/check_cocci_parse.sh @@ -0,0 +1,98 @@ +#!/bin/sh + +# If we have coccinelle installed, run try_parse.sh on every filename passed +# as an argument. If no filenames are supplied, scan a standard Tor 0.3.5 or +# later directory layout. +# +# Uses the default coccinelle exceptions file, or $TOR_COCCI_EXCEPTIONS_FILE, +# if it is set. +# +# Use TOR_COCCI_EXCEPTIONS_FILE=/dev/null check_cocci_parse.sh to disable +# the default exception file. +# +# If spatch is not installed, remind the user to install it, but exit with +# a success error status. + +scripts_cocci="$(dirname "$0")" +top="$scripts_cocci/../.." +try_parse="$scripts_cocci/try_parse.sh" + +exitcode=0 + +export TOR_COCCI_EXCEPTIONS_FILE="${TOR_COCCI_EXCEPTIONS_FILE:-$scripts_cocci/exceptions.txt}" + +PURPOSE="cocci C parsing" + +echo "Checking spatch:" + +if ! command -v spatch ; then + echo "Install coccinelle's spatch to check $PURPOSE." + exit "$exitcode" +fi + +# Returns true if $1 is greater than or equal to $2 +version_ge() +{ + if test "$1" = "$2" ; then + # return true + return 0 + fi + LOWER_VERSION="$(printf '%s\n' "$1" "$2" | $SORT_V | head -n 1)" + # implicit return + test "$LOWER_VERSION" != "$1" +} + +# 'sort -V' is a gnu extension +SORT_V="sort -V" +# Use 'sort -n' if 'sort -V' doesn't work +if ! version_ge "1" "0" ; then + echo "Your 'sort -V' command appears broken. Falling back to 'sort -n'." + echo "Some spatch version checks may give the wrong result." + SORT_V="sort -n" +fi + +# Print the full spatch version, for diagnostics +spatch --version + +MIN_SPATCH_V="1.0.4" +# This pattern needs to handle version strings like: +# spatch version 1.0.0-rc19 +# spatch version 1.0.6 compiled with OCaml version 4.05.0 +SPATCH_V=$(spatch --version | head -1 | \ + sed 's/spatch version \([0-9][^ ]*\).*/\1/') + +if ! version_ge "$SPATCH_V" "$MIN_SPATCH_V" ; then + echo "Tor requires coccinelle spatch >= $MIN_SPATCH_V to check $PURPOSE." + echo "But you have $SPATCH_V. Please install a newer version." + exit "$exitcode" +fi + +if test $# -ge 1 ; then + "$try_parse" "$@" + exitcode=$? +else + cd "$top" || exit 1 + # This is the layout in 0.3.5 + # Keep these lists consistent: + # - OWNED_TOR_C_FILES in Makefile.am + # - CHECK_FILES in pre-commit.git-hook and pre-push.git-hook + # - try_parse in check_cocci_parse.sh + "$try_parse" \ + src/lib/*/*.[ch] \ + src/core/*/*.[ch] \ + src/feature/*/*.[ch] \ + src/app/*/*.[ch] \ + src/test/*.[ch] \ + src/test/*/*.[ch] \ + src/tools/*.[ch] + exitcode=$? +fi + +if test "$exitcode" != 0 ; then + echo "Please fix these $PURPOSE errors in the above files" + echo "Set VERBOSE=1 for more details" + echo "Try running test-operator-cleanup or 'make autostyle-operators'" + echo "As a last resort, you can modify scripts/coccinelle/exceptions.txt" +fi + +exit "$exitcode" diff --git a/scripts/coccinelle/exceptions.txt b/scripts/coccinelle/exceptions.txt new file mode 100644 index 0000000000..473f4b22c5 --- /dev/null +++ b/scripts/coccinelle/exceptions.txt @@ -0,0 +1,24 @@ +# A list of exception patterns for check_cocci_parse.sh +# Passed to 'grep -f' +src/lib/cc/compat_compiler.h +src/lib/container/handles.h +src/lib/container/map.c +src/lib/container/map.h +src/lib/container/order.c +src/lib/crypt_ops/crypto_rand.c +src/lib/fs/files.h +src/lib/log/util_bug.c +src/lib/pubsub/pubsub_macros.h +src/lib/smartlist_core/smartlist_foreach.h +src/lib/testsupport/testsupport.h +src/lib/tls/tortls.h +src/lib/tls/tortls_openssl.c +src/lib/tls/x509.h +src/lib/version/version.c +src/core/mainloop/connection.c +src/core/or/reasons.c +src/feature/dirclient/dirclient.c +src/feature/nodelist/networkstatus.c +src/test/test_address.c +src/test/test_hs_cache.c +src/test/test_hs_descriptor.c diff --git a/scripts/coccinelle/test-operator-cleanup b/scripts/coccinelle/test-operator-cleanup index e7822542a4..28b4d4f588 100755 --- a/scripts/coccinelle/test-operator-cleanup +++ b/scripts/coccinelle/test-operator-cleanup @@ -1,4 +1,17 @@ #!/usr/bin/perl -w -p -i +# +# Copyright (c) 2001 Matej Pfajfar. +# Copyright (c) 2001-2004, Roger Dingledine. +# Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. +# Copyright (c) 2007-2019, The Tor Project, Inc. +# See LICENSE for licensing information + +# This script looks for instances of C comparison operators as macro arguments, +# and replaces them with our OP_* equivalents. +# +# Some macros that take operators are our tt_int_op() testing macro, and the +# standard timercmp() macro. Coccinelle can't handle their syntax, however, +# unless we give them their operators as a macro too. next if m#^ */\*# or m#^ *\* #; diff --git a/scripts/coccinelle/tor-coccinelle.h b/scripts/coccinelle/tor-coccinelle.h index 8f625dcee4..44d79325eb 100644 --- a/scripts/coccinelle/tor-coccinelle.h +++ b/scripts/coccinelle/tor-coccinelle.h @@ -1,3 +1,63 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2019, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/* + * This file looks like a C header, but its purpose is a bit different. + * + * We never include it from our real C files; we only tell Coccinelle + * about it in apply.sh. + * + * It tells the Coccinelle semantic patching tool how to understand + * things that would otherwise not be good C syntax, or which would + * otherwise not make sense to it as C. It doesn't need to produce + * semantically equivalent C, or even correct C: it only has to produce + * syntactically valid C. + */ + +#define MOCK_DECL(a, b, c) a b c #define MOCK_IMPL(a, b, c) a b c #define CHECK_PRINTF(a, b) +#define CHECK_SCANF(a, b) #define STATIC static +#define EXTERN(a,b) extern a b; + +#define STMT_BEGIN do { +#define STMT_END } while (0) + +#define BUG(x) (x) +#define IF_BUG_ONCE(x) if (x) + +#define ATTR_NORETURN +#define ATTR_UNUSED +#define ATTR_CONST +#define ATTR_MALLOC +#define ATTR_WUR +#define DISABLE_GCC_WARNING(x) +#define ENABLE_GCC_WARNING(x) + +#define HANDLE_DECL(a,b,c) +#define HANDLE_IMPL(a,b,c) +#define HT_ENTRY(x) void * +#define HT_HEAD(a,b) struct ht_head +#define HT_INITIALIZER() { } +#define X509 struct x509_st +#define STACK_OF(x) struct foo_stack_t +#define TOR_TAILQ_HEAD(a,b) struct tailq_head +#define TOR_TAILQ_ENTRY(a) struct tailq_entry +#define TOR_SIMPLEQ_HEAD(a,b) struct simpleq_entry +#define TOR_SIMPLEQ_ENTRY(a) struct simpleq_entry +#define TOR_LIST_HEAD(a,b) struct list_head +#define TOR_LIST_ENTRY(a) struct list_entry +#define TOR_SLIST_HEAD(a,b) struct slist_head +#define TOR_SLIST_ENTRY(a) struct slist_entry + +#define NS_DECL(a, b, c) a b c +#define NS(a) a + +#define CONF_TEST_MEMBERS(a,b,c) +#define DUMMY_CONF_TEST_MEMBERS + +#define EAT_SEMICOLON extern int dummy__; diff --git a/scripts/coccinelle/try_parse.sh b/scripts/coccinelle/try_parse.sh new file mode 100755 index 0000000000..a90e51b4aa --- /dev/null +++ b/scripts/coccinelle/try_parse.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +# Echo the name of every argument of this script that is not "perfect" +# according to coccinelle's --parse-c. +# +# If $TOR_COCCI_EXCEPTIONS_FILE is non-empty, skip any files that match the +# patterns in the exception file, according to "grep -f" +# +# If VERBOSE is non-empty, log spatch errors and skipped files. + +top="$(dirname "$0")/../.." + +exitcode=0 + +for fn in "$@"; do + + if test "${TOR_COCCI_EXCEPTIONS_FILE}" ; then + skip_fn=$(echo "$fn" | grep -f "${TOR_COCCI_EXCEPTIONS_FILE}") + if test "${skip_fn}" ; then + if test "${VERBOSE}" != ""; then + echo "Skipping '${skip_fn}'" + fi + continue + fi + fi + + if spatch --macro-file-builtins \ + "$top"/scripts/coccinelle/tor-coccinelle.h \ + --defined COCCI \ + --parse-c "$fn" \ + 2>/dev/null | grep "perfect = 1" > /dev/null; then + : # it's perfect + else + echo "$fn" + if test "${VERBOSE}" != ""; then + spatch --macro-file-builtins \ + "$top"/scripts/coccinelle/tor-coccinelle.h \ + --defined COCCI \ + --parse-c "$fn" + fi + exitcode=1 + fi + +done + +exit "$exitcode" diff --git a/scripts/codegen/fuzzing_include_am.py b/scripts/codegen/fuzzing_include_am.py index a944584453..aa3ba49a73 100755 --- a/scripts/codegen/fuzzing_include_am.py +++ b/scripts/codegen/fuzzing_include_am.py @@ -1,5 +1,10 @@ #!/usr/bin/python +# Future imports for Python 2.7, mandatory in 3.0 +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + FUZZERS = """ consensus descriptor diff --git a/scripts/codegen/gen_server_ciphers.py b/scripts/codegen/gen_server_ciphers.py index 5d326f8b9e..3b77952243 100755 --- a/scripts/codegen/gen_server_ciphers.py +++ b/scripts/codegen/gen_server_ciphers.py @@ -8,6 +8,11 @@ # # Run it on all the files in your openssl include directory. +# Future imports for Python 2.7, mandatory in 3.0 +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + import re import sys @@ -96,7 +101,7 @@ def parse_cipher(ciph): fwsec, = m.groups() return Ciphersuite(ciph, fwsec, "CHACHA20", "256", "POLY1305", "n/a") - print "/* Couldn't parse %s ! */"%ciph + print("/* Couldn't parse %s ! */"%ciph) return None @@ -120,12 +125,12 @@ for c in ALL_CIPHERS: colon = ' ":"' if c.name in MANDATORY: - print "%s/* Required */"%indent - print '%s%s%s'%(indent,c.name,colon) + print("%s/* Required */"%indent) + print('%s%s%s'%(indent,c.name,colon)) else: - print "#ifdef %s"%c.name - print '%s%s%s'%(indent,c.name,colon) - print "#endif" + print("#ifdef %s"%c.name) + print('%s%s%s'%(indent,c.name,colon)) + print("#endif") -print '%s;'%indent +print('%s;'%indent) diff --git a/scripts/codegen/get_mozilla_ciphers.py b/scripts/codegen/get_mozilla_ciphers.py index f23f2f1e6f..165105736a 100755 --- a/scripts/codegen/get_mozilla_ciphers.py +++ b/scripts/codegen/get_mozilla_ciphers.py @@ -10,12 +10,17 @@ # It takes two arguments: the location of a firefox source directory, and the # location of an openssl source directory. +# Future imports for Python 2.7, mandatory in 3.0 +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + import os import re import sys if len(sys.argv) != 3: - print >>sys.stderr, "Syntax: get_mozilla_ciphers.py <firefox-source-dir> <openssl-source-dir>" + print("Syntax: get_mozilla_ciphers.py <firefox-source-dir> <openssl-source-dir>", file=sys.stderr) sys.exit(1) ff_root = sys.argv[1] @@ -171,13 +176,13 @@ for fl in oSSLinclude: fp.close() # Now generate the output. -print """\ +print("""\ /* This is an include file used to define the list of ciphers clients should * advertise. Before including it, you should define the CIPHER and XCIPHER * macros. * * This file was automatically generated by get_mozilla_ciphers.py. - */""" + */""") # Go in order by the order in CipherPrefs for firefox_macro in firefox_ciphers: @@ -210,4 +215,4 @@ for firefox_macro in firefox_ciphers: #else XCIPHER(%(hex)s, %(macro)s) #endif""" % format - print res + print(res) diff --git a/scripts/codegen/makedesc.py b/scripts/codegen/makedesc.py index efca4dda9a..af926a6438 100644 --- a/scripts/codegen/makedesc.py +++ b/scripts/codegen/makedesc.py @@ -9,6 +9,11 @@ # I've used this to make inputs for unit tests. I wouldn't suggest # using it for anything else. +# Future imports for Python 2.7, mandatory in 3.0 +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + import base64 import binascii import ctypes @@ -19,12 +24,16 @@ import os import re import struct import time -import UserDict import slow_ed25519 import slownacl_curve25519 import ed25519_exts_ref +try: + xrange # Python 2 +except NameError: + xrange = range # Python 3 + # Pull in the openssl stuff we need. crypt = ctypes.CDLL(ctypes.util.find_library('crypto')) @@ -247,8 +256,8 @@ class OnDemandKeys(object): def signdesc(body, args_out=None): - rsa, ident_pem, id_digest = make_key() - _, onion_pem, _ = make_key() + rsa, ident_pem, id_digest = make_rsa_key() + _, onion_pem, _ = make_rsa_key() need_ed = '{ED25519-CERT}' in body or '{ED25519-SIGNATURE}' in body if need_ed: @@ -298,10 +307,10 @@ def signdesc(body, args_out=None): return body.rstrip() def print_c_string(ident, body): - print "static const char %s[] =" % ident + print("static const char %s[] =" % ident) for line in body.split("\n"): - print ' "%s\\n"' %(line) - print " ;" + print(' "%s\\n"' %(line)) + print(" ;") def emit_ri(name, body): info = OnDemandKeys() @@ -315,8 +324,8 @@ def emit_ei(name, body): body = info.sign_desc(body) print_c_string("EX_EI_%s"%name.upper(), body) - print 'const char EX_EI_{NAME}_FP[] = "{d.RSA_FINGERPRINT_NOSPACE}";'.format( - d=info, NAME=name.upper()) + print('const char EX_EI_{NAME}_FP[] = "{d.RSA_FINGERPRINT_NOSPACE}";'.format( + d=info, NAME=name.upper())) print_c_string("EX_EI_%s_KEY"%name.upper(), info.RSA_IDENTITY) def analyze(s): diff --git a/scripts/git/git-merge-forward.sh b/scripts/git/git-merge-forward.sh index bdd0da5b75..bbc5047cb7 100755 --- a/scripts/git/git-merge-forward.sh +++ b/scripts/git/git-merge-forward.sh @@ -11,8 +11,8 @@ function usage() echo " -n: dry run mode" echo " (default: run commands)" echo " -t: test branch mode: create new branches from the commits checked" - echo " out in each maint directory. Call these branches prefix_029," - echo " prefix_035, ... , prefix_master." + echo " out in each maint directory. Call these branches prefix_035," + echo " prefix_040, ... , prefix_master." echo " (default: merge forward maint-*, release-*, and master)" echo " -u: in test branch mode, if a prefix_* branch already exists," echo " skip creating that branch. Use after a merge error, to" @@ -87,24 +87,26 @@ TOR_WKT_NAME=${TOR_WKT_NAME:-"tor-wkt"} # New arrays need to be in the WORKTREE= array else they aren't considered. # # Only used in test branch mode -# There is no previous branch to merge forward, so the second and fifth items -# must be blank ("") -MAINT_029_TB=( "maint-0.2.9" "" "$GIT_PATH/$TOR_WKT_NAME/maint-0.2.9" \ - "_029" "") +# We create a test branch for the earliest maint branch. +# But it's the earliest maint branch, so we don't merge forward into it. +# Since we don't merge forward into it, the second and fifth items must be +# blank (""). +MAINT_035_TB=( "maint-0.3.5" "" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.5" \ + "_035" "") # Used in maint/release merge and test branch modes -MAINT_035=( "maint-0.3.5" "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.5" \ - "_035" "_029") MAINT_040=( "maint-0.4.0" "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.0" \ "_040" "_035") MAINT_041=( "maint-0.4.1" "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.1" \ "_041" "_040") -MAINT_MASTER=( "master" "maint-0.4.1" "$GIT_PATH/$TOR_MASTER_NAME" \ - "_master" "_041") +MAINT_042=( "maint-0.4.2" "maint-0.4.1" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.2" \ + "_042" "_041") +MAINT_MASTER=( "master" "maint-0.4.2" "$GIT_PATH/$TOR_MASTER_NAME" \ + "_master" "_042") -RELEASE_029=( "release-0.2.9" "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/release-0.2.9" ) RELEASE_035=( "release-0.3.5" "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.5" ) RELEASE_040=( "release-0.4.0" "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.0" ) RELEASE_041=( "release-0.4.1" "maint-0.4.1" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.1" ) +RELEASE_042=( "release-0.4.2" "maint-0.4.2" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.2" ) # The master branch path has to be the main repository thus contains the # origin that will be used to fetch the updates. All the worktrees are created @@ -113,15 +115,15 @@ ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME" # SC2034 -- shellcheck thinks that these are unused. We know better. ACTUALLY_THESE_ARE_USED=<<EOF -${MAINT_029_TB[0]} -${MAINT_035[0]} +${MAINT_035_TB[0]} ${MAINT_040[0]} ${MAINT_041[0]} +${MAINT_042[0]} ${MAINT_MASTER[0]} -${RELEASE_029[0]} ${RELEASE_035[0]} ${RELEASE_040[0]} ${RELEASE_041[0]} +${RELEASE_042[0]} EOF ####################### @@ -134,7 +136,7 @@ DRY_RUN=0 # Controlled by the -t <test-branch-prefix> option. The test branch base # name option makes git-merge-forward.sh create new test branches: -# <tbbn>_029, <tbbn>_035, ... , <tbbn>_master, and merge forward. +# <tbbn>_035, <tbbn>_040, ... , <tbbn>_master, and merge forward. TEST_BRANCH_PREFIX= # Controlled by the -u option. The use existing option checks for existing @@ -172,12 +174,11 @@ if [ -z "$TEST_BRANCH_PREFIX" ]; then # maint/release merge mode # - # List of all worktrees to work on. All defined above. Ordering is important. - # Always the maint-* branch BEFORE then the release-*. + # List of all worktrees to merge forward into. All defined above. + # Ordering is important. Always the maint-* branch BEFORE the release-*. WORKTREE=( - RELEASE_029[@] - - MAINT_035[@] + # We don't merge forward into MAINT_035_TB[@], because it's the earliest + # maint branch RELEASE_035[@] MAINT_040[@] @@ -186,21 +187,28 @@ if [ -z "$TEST_BRANCH_PREFIX" ]; then MAINT_041[@] RELEASE_041[@] + MAINT_042[@] + RELEASE_042[@] + MAINT_MASTER[@] ) else - # Test branch mode: merge to maint only, and create a new branch for 0.2.9 + # Test branch mode: base test branches on maint branches only + # + # List of all worktrees to create test branches from. All defined above. + # Ordering is important. All maint-* branches, including the earliest one. WORKTREE=( - MAINT_029_TB[@] - - MAINT_035[@] + # We want a test branch based on the earliest maint branch + MAINT_035_TB[@] MAINT_040[@] MAINT_041[@] + MAINT_042[@] + MAINT_MASTER[@] ) @@ -323,7 +331,7 @@ function merge_branch fi } -# Pull the given branch name. +# Merge origin/(branch name) into the current branch. function merge_branch_origin { local cmd="git merge --ff-only 'origin/$1'" @@ -417,9 +425,9 @@ for ((i=0; i<COUNT; i++)); do fi # Merge the previous branch into the target branch # Merge Forward Example: - # merge maint-0.2.9 into maint-0.3.5. + # merge maint-0.3.5 into maint-0.4.0. # Test Branch Example: - # merge bug99999_029 into bug99999_035. + # merge bug99999_035 into bug99999_040. # Skip the merge if the previous branch does not exist # (there's nothing to merge forward into the oldest test branch) if [ "$target_previous" ]; then diff --git a/scripts/git/git-pull-all.sh b/scripts/git/git-pull-all.sh index dc16066388..c8d115da01 100755 --- a/scripts/git/git-pull-all.sh +++ b/scripts/git/git-pull-all.sh @@ -58,16 +58,16 @@ TOR_WKT_NAME=${TOR_WKT_NAME:-"tor-wkt"} # # First set of arrays are the maint-* branch and then the release-* branch. # New arrays need to be in the WORKTREE= array else they aren't considered. -MAINT_029=( "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/maint-0.2.9" ) MAINT_035=( "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.5" ) MAINT_040=( "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.0" ) MAINT_041=( "maint-0.4.1" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.1" ) +MAINT_042=( "maint-0.4.2" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.2" ) MAINT_MASTER=( "master" "$GIT_PATH/$TOR_MASTER_NAME" ) -RELEASE_029=( "release-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/release-0.2.9" ) RELEASE_035=( "release-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.5" ) RELEASE_040=( "release-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.0" ) RELEASE_041=( "release-0.4.1" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.1" ) +RELEASE_042=( "release-0.4.2" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.2" ) # The master branch path has to be the main repository thus contains the # origin that will be used to fetch the updates. All the worktrees are created @@ -76,27 +76,23 @@ ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME" # SC2034 -- shellcheck thinks that these are unused. We know better. ACTUALLY_THESE_ARE_USED=<<EOF -${MAINT_029[0]} ${MAINT_035[0]} ${MAINT_040[0]} ${MAINT_041[0]} +${MAINT_042[0]} ${MAINT_MASTER[0]} -${RELEASE_029[0]} ${RELEASE_035[0]} ${RELEASE_040[0]} ${RELEASE_041[0]} +${RELEASE_042[0]} EOF ########################### # Git worktrees to manage # ########################### -# List of all worktrees to work on. All defined above. Ordering is important. -# Always the maint-* branch first then the release-*. +# List of all worktrees to pull. All defined above. Ordering is not important. WORKTREE=( - MAINT_029[@] - RELEASE_029[@] - MAINT_035[@] RELEASE_035[@] @@ -106,6 +102,9 @@ WORKTREE=( MAINT_041[@] RELEASE_041[@] + MAINT_042[@] + RELEASE_042[@] + MAINT_MASTER[@] ) COUNT=${#WORKTREE[@]} diff --git a/scripts/git/git-push-all.sh b/scripts/git/git-push-all.sh index 7c43fe24d8..0abddc8023 100755 --- a/scripts/git/git-push-all.sh +++ b/scripts/git/git-push-all.sh @@ -21,7 +21,7 @@ function usage() echo " -r: push to remote-name, rather than the default upstream remote." echo " (default: $DEFAULT_UPSTREAM_REMOTE, current: $UPSTREAM_REMOTE)" echo " -t: test branch mode: push test branches to remote-name. Pushes" - echo " branches prefix_029, prefix_035, ... , prefix_master." + echo " branches prefix_035, prefix_040, ... , prefix_master." echo " (default: push maint-*, release-*, and master)" echo " -s: push branches whose tips match upstream maint, release, or" echo " master branches. The default is to skip these branches," @@ -29,6 +29,8 @@ function usage() echo " CI environment failures, using code that previously passed CI." echo " (default: skip; current: $CURRENT_PUSH_SAME matching branches)" echo " --: pass further arguments to git push." + echo " All unrecognised arguments are passed to git push, but complex" + echo " arguments before -- may be mangled by getopt." echo " (default: git push --atomic, current: $GIT_PUSH)" echo echo " env vars:" @@ -95,9 +97,10 @@ PUSH_SAME=${TOR_PUSH_SAME:-0} # Argument processing # ####################### -# Controlled by the -t <test-branch-prefix> option. The test branch base -# name option makes git-merge-forward.sh create new test branches: -# <tbbn>_029, <tbbn>_035, ... , <tbbn>_master, and merge forward. +# Controlled by the -t <test-branch-prefix> option. The test branch prefix +# option makes git-merge-forward.sh create new test branches: +# <tbp>_035, <tbp>_040, ... , <tbp>_master, and merge each branch forward into +# the next one. TEST_BRANCH_PREFIX= while getopts ":hr:st:" opt; do @@ -127,9 +130,11 @@ while getopts ":hr:st:" opt; do OPTIND=$((OPTIND - 2)) ;; *) - # Assume we're done with script arguments, - # and git push will handle the option - break + # Make git push handle the option + # This might mangle options with spaces, use -- for complex options + GIT_PUSH="$GIT_PUSH $1" + shift + OPTIND=$((OPTIND - 1)) ;; esac done @@ -151,7 +156,7 @@ if [ "$TEST_BRANCH_PREFIX" ]; then fi if [ "$TOR_GIT_PUSH_PATH" ]; then - echo "Changing to $GIT_PUSH_PATH before pushing" + echo "Changing to $TOR_GIT_PUSH_PATH before pushing" cd "$TOR_GIT_PUSH_PATH" else echo "Pushing from the current directory" @@ -167,19 +172,19 @@ DEFAULT_UPSTREAM_BRANCHES= if [ "$DEFAULT_UPSTREAM_REMOTE" != "$UPSTREAM_REMOTE" ]; then DEFAULT_UPSTREAM_BRANCHES=$(echo \ "$DEFAULT_UPSTREAM_REMOTE"/master \ + "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.4.2 \ "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.4.1 \ "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.4.0 \ "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.3.5 \ - "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.2.9 \ ) fi UPSTREAM_BRANCHES=$(echo \ "$UPSTREAM_REMOTE"/master \ + "$UPSTREAM_REMOTE"/{release,maint}-0.4.2 \ "$UPSTREAM_REMOTE"/{release,maint}-0.4.1 \ "$UPSTREAM_REMOTE"/{release,maint}-0.4.0 \ "$UPSTREAM_REMOTE"/{release,maint}-0.3.5 \ - "$UPSTREAM_REMOTE"/{release,maint}-0.2.9 \ ) ######################## @@ -188,35 +193,35 @@ UPSTREAM_BRANCHES=$(echo \ PUSH_BRANCHES=$(echo \ master \ + {release,maint}-0.4.2 \ {release,maint}-0.4.1 \ {release,maint}-0.4.0 \ {release,maint}-0.3.5 \ - {release,maint}-0.2.9 \ ) if [ -z "$TEST_BRANCH_PREFIX" ]; then - # maint/release push mode + # maint/release push mode: push all branches. # # List of branches to push. Ordering is not important. PUSH_BRANCHES=$(echo \ master \ + {release,maint}-0.4.2 \ {release,maint}-0.4.1 \ {release,maint}-0.4.0 \ {release,maint}-0.3.5 \ - {release,maint}-0.2.9 \ ) else - # Test branch mode: merge to maint only, and create a new branch for 0.2.9 + # Test branch push mode: push test branches, based on each maint branch. # # List of branches to push. Ordering is not important. PUSH_BRANCHES=" \ ${TEST_BRANCH_PREFIX}_master \ + ${TEST_BRANCH_PREFIX}_042 \ ${TEST_BRANCH_PREFIX}_041 \ ${TEST_BRANCH_PREFIX}_040 \ ${TEST_BRANCH_PREFIX}_035 \ - ${TEST_BRANCH_PREFIX}_029 \ " fi @@ -224,20 +229,32 @@ fi # Entry point # ############### -# Skip the test branches that are the same as the upstream branches -if [ "$PUSH_SAME" -eq 0 ] && [ "$TEST_BRANCH_PREFIX" ]; then +if [ "$TEST_BRANCH_PREFIX" ]; then + # Skip the test branches that are the same as the default or current + # upstream branches (they have already been tested) + UPSTREAM_SKIP_SAME_AS="$UPSTREAM_BRANCHES $DEFAULT_UPSTREAM_BRANCHES" +else + # Skip the local maint-*, release-*, master branches that are the same as the + # current upstream branches, but ignore the default upstream + # (we want to update a non-default remote, even if it matches the default) + UPSTREAM_SKIP_SAME_AS="$UPSTREAM_BRANCHES" +fi + +# Skip branches that match the relevant upstream(s) +if [ "$PUSH_SAME" -eq 0 ]; then NEW_PUSH_BRANCHES= for b in $PUSH_BRANCHES; do PUSH_COMMIT=$(git rev-parse "$b") SKIP_UPSTREAM= - for u in $DEFAULT_UPSTREAM_BRANCHES $UPSTREAM_BRANCHES; do - UPSTREAM_COMMIT=$(git rev-parse "$u") + for u in $UPSTREAM_SKIP_SAME_AS; do + # Skip the branch check on error + UPSTREAM_COMMIT=$(git rev-parse "$u" 2>/dev/null) || continue if [ "$PUSH_COMMIT" = "$UPSTREAM_COMMIT" ]; then SKIP_UPSTREAM="$u" fi done if [ "$SKIP_UPSTREAM" ]; then - printf "Skipping unchanged: %s remote: %s\\n" \ + printf "Skipping unchanged: %s matching remote: %s\\n" \ "$b" "$SKIP_UPSTREAM" else if [ "$NEW_PUSH_BRANCHES" ]; then @@ -250,6 +267,12 @@ if [ "$PUSH_SAME" -eq 0 ] && [ "$TEST_BRANCH_PREFIX" ]; then PUSH_BRANCHES=${NEW_PUSH_BRANCHES} fi +if [ ! "$PUSH_BRANCHES" ]; then + echo "No branches to push!" + # We expect the rest of the script to run without errors, even if there + # are no branches +fi + if [ "$PUSH_DELAY" -le 0 ]; then echo "Pushing $PUSH_BRANCHES" # We know that there are no spaces in any branch within $PUSH_BRANCHES, so @@ -262,28 +285,43 @@ if [ "$PUSH_DELAY" -le 0 ]; then else # Push the branches in optimal CI order, with a delay between each push PUSH_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | sort -V) - MASTER_BRANCH=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep master) + MASTER_BRANCH=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep master) \ + || true # Skipped master branch if [ -z "$TEST_BRANCH_PREFIX" ]; then - MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep maint) + MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep maint) \ + || true # Skipped all maint branches RELEASE_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep release | \ - tr "\\n" " ") - printf \ - "Pushing with %ss delays, so CI runs in this order:\\n%s\\n%s\\n%s\\n" \ - "$PUSH_DELAY" "$MASTER_BRANCH" "$MAINT_BRANCHES" "$RELEASE_BRANCHES" + tr "\\n" " ") || true # Skipped all release branches else # Actually test branches based on maint branches - MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep -v master) - printf "Pushing with %ss delays, so CI runs in this order:\\n%s\\n%s\\n" \ - "$PUSH_DELAY" "$MASTER_BRANCH" "$MAINT_BRANCHES" + MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep -v master) \ + || true # Skipped all maint test branches # No release branches RELEASE_BRANCHES= fi - $GIT_PUSH "$@" "$UPSTREAM_REMOTE" "$MASTER_BRANCH" - sleep "$PUSH_DELAY" + if [ "$MASTER_BRANCH" ] || [ "$MAINT_BRANCHES" ] \ + || [ "$RELEASE_BRANCHES" ]; then + printf "Pushing with %ss delays, so CI runs in this order:\\n" \ + "$PUSH_DELAY" + if [ "$MASTER_BRANCH" ]; then + printf "%s\\n" "$MASTER_BRANCH" + fi + if [ "$MAINT_BRANCHES" ]; then + printf "%s\\n" "$MAINT_BRANCHES" + fi + if [ "$RELEASE_BRANCHES" ]; then + printf "%s\\n" "$RELEASE_BRANCHES" + fi + fi # shellcheck disable=SC2086 - for b in $MAINT_BRANCHES; do + for b in $MASTER_BRANCH $MAINT_BRANCHES; do $GIT_PUSH "$@" "$UPSTREAM_REMOTE" "$b" - sleep "$PUSH_DELAY" + # If we are pushing more than one branch, delay. + # In the unlikely scenario where we are pushing maint without master, + # or maint without release, there may be an extra delay + if [ "$MAINT_BRANCHES" ] || [ "$RELEASE_BRANCHES" ]; then + sleep "$PUSH_DELAY" + fi done if [ "$RELEASE_BRANCHES" ]; then # shellcheck disable=SC2086 diff --git a/scripts/git/git-setup-dirs.sh b/scripts/git/git-setup-dirs.sh new file mode 100755 index 0000000000..20a148204a --- /dev/null +++ b/scripts/git/git-setup-dirs.sh @@ -0,0 +1,550 @@ +#!/usr/bin/env bash + +SCRIPT_NAME=$(basename "$0") + +function usage() +{ + echo "$SCRIPT_NAME [-h] [-n] [-u]" + echo + echo " arguments:" + echo " -h: show this help text" + echo " -n: dry run mode" + echo " (default: run commands)" + echo " -u: if a directory or worktree already exists, use it" + echo " (default: fail and exit on existing directories)" + echo + echo " env vars:" + echo " required:" + echo " TOR_FULL_GIT_PATH: where the git repository directories reside." + echo " You must set this env var, we recommend \$HOME/git/" + echo " (default: fail if this env var is not set;" + echo " current: $GIT_PATH)" + echo + echo " optional:" + echo " TOR_MASTER: the name of the directory containing the tor.git clone" + echo " The tor master git directory is \$GIT_PATH/\$TOR_MASTER" + echo " (default: tor; current: $TOR_MASTER_NAME)" + echo " TOR_WKT_NAME: the name of the directory containing the tor" + echo " worktrees. The tor worktrees are:" + echo " \$GIT_PATH/\$TOR_WKT_NAME/{maint-*,release-*}" + echo " (default: tor-wkt; current: $TOR_WKT_NAME)" + echo " TOR_GIT_ORIGIN_PULL: the origin remote pull URL." + echo " (current: $GIT_ORIGIN_PULL)" + echo " TOR_GIT_ORIGIN_PUSH: the origin remote push URL" + echo " (current: $GIT_ORIGIN_PUSH)" + echo " TOR_UPSTREAM_REMOTE_NAME: the default upstream remote." + echo " If \$TOR_UPSTREAM_REMOTE_NAME is not 'origin', we have a" + echo " separate upstream remote, and we don't push to origin." + echo " (default: $DEFAULT_UPSTREAM_REMOTE)" + echo " TOR_GITHUB_PULL: the tor-github remote pull URL" + echo " (current: $GITHUB_PULL)" + echo " TOR_GITHUB_PUSH: the tor-github remote push URL" + echo " (current: $GITHUB_PUSH)" + echo " TOR_EXTRA_CLONE_ARGS: extra arguments to git clone" + echo " (current: $TOR_EXTRA_CLONE_ARGS)" + echo " TOR_EXTRA_REMOTE_NAME: the name of an extra remote" + echo " This remote is not pulled by this script or git-pull-all.sh." + echo " This remote is not pushed by git-push-all.sh." + echo " (current: $TOR_EXTRA_REMOTE_NAME)" + echo " TOR_EXTRA_REMOTE_PULL: the extra remote pull URL." + echo " (current: $TOR_EXTRA_REMOTE_PULL)" + echo " TOR_EXTRA_REMOTE_PUSH: the extra remote push URL" + echo " (current: $TOR_EXTRA_REMOTE_PUSH)" + echo " we recommend that you set these env vars in your ~/.profile" +} + +################# +# Configuration # +################# + +# Don't change this configuration - set the env vars in your .profile + +# Where are all those git repositories? +GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"} +# The tor master git repository directory from which all the worktree have +# been created. +TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"} +# The worktrees location (directory). +TOR_WKT_NAME=${TOR_WKT_NAME:-"tor-wkt"} + +# Origin repositories +GIT_ORIGIN_PULL=${TOR_GIT_ORIGIN_PULL:-"https://git.torproject.org/tor.git"} +GIT_ORIGIN_PUSH=${TOR_GIT_ORIGIN_PUSH:-"git@git-rw.torproject.org:tor.git"} +# The upstream remote which git.torproject.org/tor.git points to. +DEFAULT_UPSTREAM_REMOTE=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"} +# Copy the URLs from origin +GIT_UPSTREAM_PULL="$GIT_ORIGIN_PULL" +GIT_UPSTREAM_PUSH="$GIT_ORIGIN_PUSH" +# And avoid pushing to origin if we have an upstream +if [ "$DEFAULT_UPSTREAM_REMOTE" != "origin" ]; then + GIT_ORIGIN_PUSH="No pushes to origin, if there is an upstream" +fi +# GitHub repositories +GITHUB_PULL=${TOR_GITHUB_PULL:-"https://github.com/torproject/tor.git"} +GITHUB_PUSH=${TOR_GITHUB_PUSH:-"No_Pushing_To_GitHub"} + +########################## +# Git branches to manage # +########################## + +# The branches and worktrees need to be modified when there is a new branch, +# and when an old branch is no longer supported. + +# Configuration of the branches that needs merging. The values are in order: +# (0) current maint/release branch name +# (1) Full path of the git worktree +# +# First set of arrays are the maint-* branch and then the release-* branch. +# New arrays need to be in the WORKTREE= array else they aren't considered. +MAINT_035=( "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.5" ) +MAINT_040=( "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.0" ) +MAINT_041=( "maint-0.4.1" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.1" ) +MAINT_042=( "maint-0.4.2" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.2" ) +MAINT_MASTER=( "master" "$GIT_PATH/$TOR_MASTER_NAME" ) + +RELEASE_035=( "release-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.5" ) +RELEASE_040=( "release-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.0" ) +RELEASE_041=( "release-0.4.1" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.1" ) +RELEASE_042=( "release-0.4.2" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.2" ) + +# The master branch path has to be the main repository thus contains the +# origin that will be used to fetch the updates. All the worktrees are created +# from that repository. +ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME" + +# SC2034 -- shellcheck thinks that these are unused. We know better. +ACTUALLY_THESE_ARE_USED=<<EOF +${MAINT_035[0]} +${MAINT_040[0]} +${MAINT_041[0]} +${MAINT_042[0]} +${MAINT_MASTER[0]} +${RELEASE_035[0]} +${RELEASE_040[0]} +${RELEASE_041[0]} +${RELEASE_042[0]} +EOF + +####################### +# Argument processing # +####################### + +# Controlled by the -n option. The dry run option will just output the command +# that would have been executed for each worktree. +DRY_RUN=0 + +# Controlled by the -s option. The use existing option checks for existing +# directories, and re-uses them, rather than creating a new directory. +USE_EXISTING=0 +USE_EXISTING_HINT="Use existing: '$SCRIPT_NAME -u'." + +while getopts "hnu" opt; do + case "$opt" in + h) usage + exit 0 + ;; + n) DRY_RUN=1 + echo " *** DRY RUN MODE ***" + ;; + u) USE_EXISTING=1 + echo " *** USE EXISTING DIRECTORIES MODE ***" + ;; + *) + echo + usage + exit 1 + ;; + esac +done + +########################### +# Git worktrees to manage # +########################### + +WORKTREE=( + MAINT_035[@] + RELEASE_035[@] + + MAINT_040[@] + RELEASE_040[@] + + MAINT_041[@] + RELEASE_041[@] + + MAINT_042[@] + RELEASE_042[@] + + MAINT_MASTER[@] +) + +COUNT=${#WORKTREE[@]} + +############# +# Constants # +############# + +# Control characters +CNRM=$'\x1b[0;0m' # Clear color + +# Bright color +BGRN=$'\x1b[1;32m' +BBLU=$'\x1b[1;34m' +BRED=$'\x1b[1;31m' +BYEL=$'\x1b[1;33m' +IWTH=$'\x1b[3;37m' + +# Strings for the pretty print. +MARKER="${BBLU}[${BGRN}+${BBLU}]${CNRM}" +SUCCESS="${BGRN}success${CNRM}" +SKIPPED="${BYEL}skipped${CNRM}" +FAILED="${BRED}failed${CNRM}" + +#################### +# Helper functions # +#################### + +# Validate the given returned value (error code), print success or failed. The +# second argument is the error output in case of failure, it is printed out. +# On failure, this function exits. +function validate_ret +{ + if [ "$1" -eq 0 ]; then + printf "%s\\n" "$SUCCESS" + else + printf "%s\\n" "$FAILED" + printf " %s\\n" "$2" + exit 1 + fi +} + +# Validate the given returned value (error code), print success, skipped, or +# failed. If $USE_EXISTING is 0, fail on error, otherwise, skip on error. +# The second argument is the error output in case of failure, it is printed +# out. On failure, this function exits. +function validate_ret_skip +{ + if [ "$1" -ne 0 ]; then + if [ "$USE_EXISTING" -eq "0" ]; then + # Fail and exit with error + validate_ret "$1" "$2 $USE_EXISTING_HINT" + else + printf "%s\\n" "$SKIPPED" + printf " %s\\n" "${IWTH}$2${CNRM}" + # Tell the caller to skip the rest of the function + return 0 + fi + fi + # Tell the caller to continue + return 1 +} + +# Create a directory, and any missing enclosing directories. +# If the directory already exists: fail if $USE_EXISTING is 0, otherwise skip. +function make_directory +{ + local cmd="mkdir -p '$1'" + printf " %s Creating directory %s..." "$MARKER" "$1" + local check_cmd="[ ! -d '$1' ]" + msg=$( eval "$check_cmd" 2>&1 ) + if validate_ret_skip $? "Directory already exists."; then + return + fi + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Create a symlink from the first argument to the second argument +# If the link already exists: fail if $USE_EXISTING is 0, otherwise skip. +function make_symlink +{ + local cmd="ln -s '$1' '$2'" + printf " %s Creating symlink from %s to %s..." "$MARKER" "$1" "$2" + local check_cmd="[ ! -e '$2' ]" + msg=$( eval "$check_cmd" 2>&1 ) + if validate_ret_skip $? "File already exists."; then + return + fi + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Go into the directory or repository, even if $DRY_RUN is non-zero. +# If the directory does not exist, fail and log an error. +# Otherwise, silently succeed. +function goto_dir +{ + if ! cd "$1" 1>/dev/null 2>/dev/null ; then + printf " %s Changing to directory %s..." "$MARKER" "$1" + validate_ret 1 "$1: Not found. Stopping." + fi +} + +# Clone a repository into a directory. +# If the directory already exists: fail if $USE_EXISTING is 0, otherwise skip. +function clone_repo +{ + local cmd="git clone $TOR_EXTRA_CLONE_ARGS '$1' '$2'" + printf " %s Cloning %s into %s..." "$MARKER" "$1" "$2" + local check_cmd="[ ! -d '$2' ]" + msg=$( eval "$check_cmd" 2>&1 ) + if validate_ret_skip $? "Directory already exists."; then + # If we skip the clone, we need to do a fetch + goto_dir "$ORIGIN_PATH" + fetch_remote "origin" + return + fi + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Add a remote by name and URL. +# If the remote already exists: fail if $USE_EXISTING is 0, otherwise skip. +function add_remote +{ + local cmd="git remote add '$1' '$2'" + printf " %s Adding remote %s at %s..." "$MARKER" "$1" "$2" + local check_cmd="git remote get-url '$1'" + msg=$( eval "$check_cmd" 2>&1 ) + ret=$? + # We don't want a remote, so we invert the exit status + if validate_ret_skip $(( ! ret )) \ + "Remote already exists for $1 at $msg."; then + return + fi + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Set a remote's push URL by name and URL. +function set_remote_push +{ + local cmd="git remote set-url --push '$1' '$2'" + printf " %s Setting remote %s push URL to '%s'..." "$MARKER" "$1" "$2" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Fetch a remote by name. +function fetch_remote +{ + local cmd="git fetch '$1'" + printf " %s Fetching %s..." "$MARKER" "$1" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Replace the fetch configs for a remote with config if they match a pattern. +function replace_fetch_config +{ + local cmd="git config --replace-all remote.'$1'.fetch '$2' '$3'" + printf " %s Replacing %s fetch configs for '%s'..." \ + "$MARKER" "$1" "$3" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Set up the tor-github PR config, so tor-github/pr/NNNN/head points to GitHub +# PR NNNN. In some repositories, "/head" is optional. +function set_tor_github_pr_fetch_config +{ + # Standard branches + replace_fetch_config tor-github \ + "+refs/heads/*:refs/remotes/tor-github/*" \ + "refs/heads" + # PRs + replace_fetch_config "tor-github" \ + "+refs/pull/*:refs/remotes/tor-github/pr/*" \ + "refs/pull.*pr" +} + +# Add a new worktree for branch at path. +# If the directory already exists: fail if $USE_EXISTING is 0, otherwise skip. +function add_worktree +{ + local cmd="git worktree add '$2' '$1'" + printf " %s Adding worktree for %s at %s..." "$MARKER" "$1" "$2" + local check_cmd="[ ! -d '$2' ]" + msg=$( eval "$check_cmd" 2>&1 ) + if validate_ret_skip $? "Directory already exists."; then + return + fi + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Switch to the given branch name. +# If the branch does not exist: fail. +function switch_branch +{ + local cmd="git checkout '$1'" + printf " %s Switching branch to %s..." "$MARKER" "$1" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Checkout a new branch with the given branch name. +# If the branch already exists: fail if $USE_EXISTING is 0, otherwise skip. +function new_branch +{ + local cmd="git checkout -b '$1'" + printf " %s Creating new branch %s..." "$MARKER" "$1" + local check_cmd="git branch --list '$1'" + msg=$( eval "$check_cmd" 2>&1 ) + if validate_ret_skip $? "Branch already exists."; then + return + fi + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +# Switch to an existing branch, or checkout a new branch with the given +# branch name. +function switch_or_new_branch +{ + local cmd="git rev-parse --verify '$1'" + if [ $DRY_RUN -eq 0 ]; then + # Call switch_branch if there is a branch, or new_branch if there is not + msg=$( eval "$cmd" 2>&1 ) + RET=$? + if [ $RET -eq 0 ]; then + # Branch: (commit id) + switch_branch "$1" + elif [ $RET -eq 128 ]; then + # Not a branch: "fatal: Needed a single revision" + new_branch "$1" + else + # Unexpected return value + validate_ret $RET "$msg" + fi + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}, then depending on the result:" + switch_branch "$1" + new_branch "$1" + fi +} + +# Set the upstream for branch to upstream. +function set_upstream +{ + # Note the argument order is swapped + local cmd="git branch --set-upstream-to='$2' '$1'" + printf " %s Setting upstream for %s to %s..." "$MARKER" "$1" "$2" + if [ $DRY_RUN -eq 0 ]; then + msg=$( eval "$cmd" 2>&1 ) + validate_ret $? "$msg" + else + printf "\\n %s\\n" "${IWTH}$cmd${CNRM}" + fi +} + +############### +# Entry point # +############### + +printf "%s Setting up the repository and remote %s\\n" "$MARKER" \ + "${BYEL}origin${CNRM}" +# First, fetch the origin. +ORIGIN_PARENT=$(dirname "$ORIGIN_PATH") +make_directory "$ORIGIN_PARENT" +# This is just cd with an error check +goto_dir "$ORIGIN_PARENT" + +# clone repository / origin remote +clone_repo "$GIT_ORIGIN_PULL" "$TOR_MASTER_NAME" +goto_dir "$ORIGIN_PATH" +set_remote_push "origin" "$GIT_ORIGIN_PUSH" + +# upstream remote, if different to origin +if [ "$DEFAULT_UPSTREAM_REMOTE" != "origin" ]; then + printf "%s Setting up remote %s\\n" "$MARKER" \ + "${BYEL}$DEFAULT_UPSTREAM_REMOTE${CNRM}" + add_remote "$DEFAULT_UPSTREAM_REMOTE" "$GIT_UPSTREAM_PULL" + set_remote_push "$DEFAULT_UPSTREAM_REMOTE" "$GIT_UPSTREAM_PUSH" + fetch_remote "$DEFAULT_UPSTREAM_REMOTE" +fi + +# GitHub remote +printf "%s Setting up remote %s\\n" "$MARKER" "${BYEL}tor-github${CNRM}" +# Add remote +add_remote "tor-github" "$GITHUB_PULL" +set_remote_push "tor-github" "$GITHUB_PUSH" +# Add custom fetch for PRs +set_tor_github_pr_fetch_config +# Now fetch them all +fetch_remote "tor-github" + +# Extra remote +if [ "$TOR_EXTRA_REMOTE_NAME" ]; then + printf "%s Setting up remote %s\\n" "$MARKER" \ + "${BYEL}$TOR_EXTRA_REMOTE_NAME${CNRM}" + # Add remote + add_remote "$TOR_EXTRA_REMOTE_NAME" "$TOR_EXTRA_REMOTE_PULL" + set_remote_push "$TOR_EXTRA_REMOTE_NAME" "$TOR_EXTRA_REMOTE_PUSH" + # But leave it to the user to decide if they want to fetch it + #fetch_remote "$TOR_EXTRA_REMOTE_NAME" +fi + +# Go over all configured worktree. +for ((i=0; i<COUNT; i++)); do + branch=${!WORKTREE[$i]:0:1} + repo_path=${!WORKTREE[$i]:1:1} + + printf "%s Handling branch %s\\n" "$MARKER" "${BYEL}$branch${CNRM}" + # We cloned the repository, and master is the default branch + if [ "$branch" = "master" ]; then + if [ "$TOR_MASTER_NAME" != "master" ]; then + # Set up a master link in the worktree directory + make_symlink "$repo_path" "$GIT_PATH/$TOR_WKT_NAME/master" + fi + else + # git makes worktree directories if they don't exist + add_worktree "origin/$branch" "$repo_path" + fi + goto_dir "$repo_path" + switch_or_new_branch "$branch" + set_upstream "$branch" "origin/$branch" +done + +echo +echo "Remember to copy the git hooks from tor/scripts/git/*.git-hook to" +echo "$ORIGIN_PATH/.git/hooks/*" diff --git a/scripts/git/pre-commit.git-hook b/scripts/git/pre-commit.git-hook index 1c381ec60a..f630a242bd 100755 --- a/scripts/git/pre-commit.git-hook +++ b/scripts/git/pre-commit.git-hook @@ -13,30 +13,67 @@ cd "$workdir" || exit 1 set -e +if [ $# -eq 0 ]; then + # When called in pre-commit, check the files modified in this commit + CHECK_FILTER="git diff --cached --name-only --diff-filter=ACMR" + # Use the appropriate owned tor source list to filter the changed files + + # This is the layout in 0.3.5 and later. + + # Keep these lists consistent: + # - OWNED_TOR_C_FILES in Makefile.am + # - CHECK_FILES in pre-commit.git-hook and pre-push.git-hook + # - try_parse in check_cocci_parse.sh + CHECK_FILES="$($CHECK_FILTER \ + src/lib/*/*.[ch] \ + src/core/*/*.[ch] \ + src/feature/*/*.[ch] \ + src/app/*/*.[ch] \ + src/test/*.[ch] \ + src/test/*/*.[ch] \ + src/tools/*.[ch] \ + )" +else + # When called in pre-push, concatenate the argument array + # Fails on special characters in file names + CHECK_FILES="$*" +fi + +## General File Checks + if [ -n "$(ls ./changes/)" ]; then python scripts/maint/lintChanges.py ./changes/* fi -if [ -d src/lib ]; then - # This is the layout in 0.3.5 - perl scripts/maint/checkSpace.pl -C \ - src/lib/*/*.[ch] \ - src/core/*/*.[ch] \ - src/feature/*/*.[ch] \ - src/app/*/*.[ch] \ - src/test/*.[ch] \ - src/test/*/*.[ch] \ - src/tools/*.[ch] -elif [ -d src/common ]; then - # This was the layout before 0.3.5 - perl scripts/maint/checkSpace.pl -C \ - src/common/*/*.[ch] \ - src/or/*/*.[ch] \ - src/test/*.[ch] \ - src/test/*/*.[ch] \ - src/tools/*.[ch] +if [ -e scripts/maint/checkShellScripts.sh ]; then + scripts/maint/checkShellScripts.sh +fi + +# Always run the practracker unit tests +PT_DIR=scripts/maint/practracker + +if [ -e "${PT_DIR}/test_practracker.sh" ]; then + "${PT_DIR}/test_practracker.sh" fi +if [ -e scripts/maint/checkSpaceTest.sh ]; then + scripts/maint/checkSpaceTest.sh +fi + +if [ ! "$CHECK_FILES" ]; then + echo "No modified tor-owned source files, skipping further checks" + exit 0 +fi + +## Owned Source File Checks + +printf "Modified tor-owned source files:\\n%s\\n" "$CHECK_FILES" + +# We want word splitting here, because file names are space separated +# shellcheck disable=SC2086 +perl scripts/maint/checkSpace.pl -C \ + $CHECK_FILES + if test -e scripts/maint/practracker/includes.py; then python scripts/maint/practracker/includes.py fi @@ -44,16 +81,19 @@ fi # Only call practracker if ${PT_DIR}/.enable_practracker_in_hooks exists # We do this check so that we can enable practracker in hooks in master, and # disable it on maint branches -PT_DIR=scripts/maint/practracker - if [ -e "${PT_DIR}/practracker.py" ]; then if [ -e "${PT_DIR}/.enable_practracker_in_hooks" ]; then - if ! python3 "${PT_DIR}/practracker.py" "$workdir"; then - exit 1 - fi + python3 "${PT_DIR}/practracker.py" "$workdir" fi fi -if [ -e scripts/maint/checkShellScripts.sh ]; then - scripts/maint/checkShellScripts.sh +if [ -e scripts/coccinelle/check_cocci_parse.sh ]; then + + # Run a verbose cocci parse check on the changed files + # (spatch is slow, so we don't want to check all the files.) + # + # We want word splitting here, because file names are space separated + # shellcheck disable=SC2086 + VERBOSE=1 scripts/coccinelle/check_cocci_parse.sh \ + $CHECK_FILES fi diff --git a/scripts/git/pre-push.git-hook b/scripts/git/pre-push.git-hook index f4504c4215..7b06f3734d 100755 --- a/scripts/git/pre-push.git-hook +++ b/scripts/git/pre-push.git-hook @@ -16,91 +16,109 @@ # The following sample script was used as starting point: # https://github.com/git/git/blob/master/templates/hooks--pre-push.sample +# Are you adding a new check to the git hooks? +# - Common checks belong in the pre-commit hook +# - Push-only checks belong in the pre-push hook + echo "Running pre-push hook" z40=0000000000000000000000000000000000000000 upstream_name=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"} -# Are you adding a new check to the git hooks? -# - Common checks belong in the pre-commit hook -# - Push-only checks belong in the pre-push hook -# -# Call the pre-commit hook for the common checks, if it is executable. workdir=$(git rev-parse --show-toplevel) -if [ -x "$workdir/.git/hooks/pre-commit" ]; then - if ! "$workdir"/.git/hooks/pre-commit; then - exit 1 - fi -fi -remote="$1" +cd "$workdir" || exit 1 +remote="$1" remote_name=$(git remote --verbose | grep "$2" | awk '{print $1}' | head -n 1) -if [[ "$remote_name" != "$upstream_name" ]]; then - echo "Not pushing to upstream - refraining from further checks" - exit 0 -fi ref_is_upstream_branch() { - if [ "$1" == "refs/heads/master" ] || - [[ "$1" == refs/heads/release-* ]] || - [[ "$1" == refs/heads/maint-* ]] - then - return 1 - fi + if [ "$1" == "refs/heads/master" ] || + [[ "$1" == refs/heads/release-* ]] || + [[ "$1" == refs/heads/maint-* ]]; then + return 1 + fi } # shellcheck disable=SC2034 while read -r local_ref local_sha remote_ref remote_sha do - if [ "$local_sha" = $z40 ] - then - # Handle delete - : - else - if [ "$remote_sha" = $z40 ] - then - # New branch, examine all commits - range="$local_sha" - else - # Update to existing branch, examine new commits - range="$remote_sha..$local_sha" - fi - - if (ref_is_upstream_branch "$local_ref" == 0 || - ref_is_upstream_branch "$remote_ref" == 0) && - [ "$local_ref" != "$remote_ref" ] - then - if [ "$remote" == "origin" ] - then - echo >&2 "Not pushing: $local_ref to $remote_ref" - echo >&2 "If you really want to push this, use --no-verify." - exit 1 - else - continue - fi - fi - - # Check for fixup! commit - commit=$(git rev-list -n 1 --grep '^fixup!' "$range") - if [ -n "$commit" ] - then - echo >&2 "Found fixup! commit in $local_ref, not pushing" - echo >&2 "If you really want to push this, use --no-verify." - exit 1 - fi - - # Check for squash! commit - commit=$(git rev-list -n 1 --grep '^squash!' "$range") - if [ -n "$commit" ] - then - echo >&2 "Found squash! commit in $local_ref, not pushing" - echo >&2 "If you really want to push this, use --no-verify." - exit 1 - fi - fi + if [ "$local_sha" = $z40 ]; then + # Handle delete + : + else + if [ "$remote_sha" = $z40 ]; then + # New branch, examine commits not in master + range="master...$local_sha" + else + # Update to existing branch, examine new commits + range="$remote_sha..$local_sha" + fi + + # Call the pre-commit hook for the common checks, if it is executable + if [ -x scripts/git/pre-commit.git-hook ]; then + # Only check the files newly modified in this branch + CHECK_FILTER="git diff --name-only --diff-filter=ACMR $range" + # Use the appropriate owned tor source list to filter the changed + # files + # This is the layout in 0.3.5 + # Keep these lists consistent: + # - OWNED_TOR_C_FILES in Makefile.am + # - CHECK_FILES in pre-commit.git-hook and pre-push.git-hook + # - try_parse in check_cocci_parse.sh + CHECK_FILES="$($CHECK_FILTER \ + src/lib/*/*.[ch] \ + src/core/*/*.[ch] \ + src/feature/*/*.[ch] \ + src/app/*/*.[ch] \ + src/test/*.[ch] \ + src/test/*/*.[ch] \ + src/tools/*.[ch] \ + )" + + # We want word splitting here, because file names are space + # separated + # shellcheck disable=SC2086 + if ! scripts/git/pre-commit.git-hook $CHECK_FILES ; then + exit 1 + fi + fi + + if [[ "$remote_name" != "$upstream_name" ]]; then + echo "Not pushing to upstream - refraining from further checks" + continue + fi + + if (ref_is_upstream_branch "$local_ref" == 0 || + ref_is_upstream_branch "$remote_ref" == 0) && + [ "$local_ref" != "$remote_ref" ]; then + if [ "$remote" == "origin" ]; then + echo >&2 "Not pushing: $local_ref to $remote_ref" + echo >&2 "If you really want to push this, use --no-verify." + exit 1 + else + continue + fi + fi + + # Check for fixup! commit + commit=$(git rev-list -n 1 --grep '^fixup!' "$range") + if [ -n "$commit" ]; then + echo >&2 "Found fixup! commit in $local_ref, not pushing" + echo >&2 "If you really want to push this, use --no-verify." + exit 1 + fi + + # Check for squash! commit + commit=$(git rev-list -n 1 --grep '^squash!' "$range") + if [ -n "$commit" ]; then + echo >&2 "Found squash! commit in $local_ref, not pushing" + echo >&2 "If you really want to push this, use --no-verify." + exit 1 + fi + fi done exit 0 diff --git a/scripts/maint/add_c_file.py b/scripts/maint/add_c_file.py index adf7ce79bb..e1e224d8d5 100755 --- a/scripts/maint/add_c_file.py +++ b/scripts/maint/add_c_file.py @@ -4,53 +4,89 @@ Add a C file with matching header to the Tor codebase. Creates both files from templates, and adds them to the right include.am file. + This script takes paths relative to the top-level tor directory. It + expects to be run from that directory. + + This script creates files, and inserts them into include.am, also + relative to the top-level tor directory. + + But the template content in those files is relative to tor's src + directory. (This script strips "src" from the paths used to create + templated comments and macros.) + + This script expects posix paths, so it should be run with a python + where os.path is posixpath. (Rather than ntpath.) This probably means + Linux, macOS, or BSD, although it might work on Windows if your python + was compiled with mingw, MSYS, or cygwin. + Example usage: % add_c_file.py ./src/feature/dirauth/ocelot.c """ +# Future imports for Python 2.7, mandatory in 3.0 +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + import os import re import time -def topdir_file(name): - """Strip opening "src" from a filename""" - if name.startswith("src/"): - name = name[4:] - return name - -def guard_macro(name): - """Return the guard macro that should be used for the header file 'name'. +def tordir_file(fname): + """Make fname relative to the current directory, which should be the + top-level tor directory. Also performs basic path simplifications.""" + return os.path.normpath(os.path.relpath(fname)) + +def srcdir_file(tor_fname): + """Make tor_fname relative to tor's "src" directory. + Also performs basic path simplifications. + (This function takes paths relative to the top-level tor directory, + but outputs a path that is relative to tor's src directory.)""" + return os.path.normpath(os.path.relpath(tor_fname, 'src')) + +def guard_macro(src_fname): + """Return the guard macro that should be used for the header file + 'src_fname'. This function takes paths relative to tor's src directory. """ - td = topdir_file(name).replace(".", "_").replace("/", "_").upper() + td = src_fname.replace(".", "_").replace("/", "_").upper() return "TOR_{}".format(td) -def makeext(name, new_extension): - """Replace the extension for the file called 'name' with 'new_extension'. +def makeext(fname, new_extension): + """Replace the extension for the file called 'fname' with 'new_extension'. + This function takes and returns paths relative to either the top-level + tor directory, or tor's src directory, and returns the same kind + of path. """ - base = os.path.splitext(name)[0] + base = os.path.splitext(fname)[0] return base + "." + new_extension -def instantiate_template(template, output_fname): +def instantiate_template(template, tor_fname): """ Fill in a template with string using the fields that should be used - for 'output_fname'. + for 'tor_fname'. + + This function takes paths relative to the top-level tor directory, + but the paths in the completed template are relative to tor's src + directory. (Except for one of the fields, which is just a basename). """ + src_fname = srcdir_file(tor_fname) names = { # The relative location of the header file. - 'header_path' : makeext(topdir_file(output_fname), "h"), + 'header_path' : makeext(src_fname, "h"), # The relative location of the C file file. - 'c_file_path' : makeext(topdir_file(output_fname), "c"), + 'c_file_path' : makeext(src_fname, "c"), # The truncated name of the file. - 'short_name' : os.path.basename(output_fname), + 'short_name' : os.path.basename(src_fname), # The current year, for the copyright notice 'this_year' : time.localtime().tm_year, # An appropriate guard macro, for the header. - 'guard_macro' : guard_macro(output_fname), + 'guard_macro' : guard_macro(src_fname), } return template.format(**names) +# This template operates on paths relative to tor's src directory HEADER_TEMPLATE = """\ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. @@ -69,6 +105,7 @@ HEADER_TEMPLATE = """\ #endif /* !defined({guard_macro}) */ """ +# This template operates on paths relative to the tor's src directory C_FILE_TEMPLATE = """\ /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. @@ -90,16 +127,22 @@ class AutomakeChunk: Represents part of an automake file. If it is decorated with an ADD_C_FILE comment, it has a "kind" based on what to add to it. Otherwise, it only has a bunch of lines in it. + + This class operates on paths relative to the top-level tor directory. """ pat = re.compile(r'# ADD_C_FILE: INSERT (\S*) HERE', re.I) def __init__(self): self.lines = [] self.kind = "" + self.hasBlank = False # true if we end with a blank line. def addLine(self, line): """ Insert a line into this chunk while parsing the automake file. + + Return True if we have just read the last line in the chunk, and + False otherwise. """ m = self.pat.match(line) if m: @@ -107,23 +150,28 @@ class AutomakeChunk: raise ValueError("control line not preceded by a blank line") self.kind = m.group(1) - self.lines.append(line) if line.strip() == "": + self.hasBlank = True return True + self.lines.append(line) + return False - def insertMember(self, member): + def insertMember(self, new_tor_fname): """ - Add a new member to this chunk. Try to insert it in alphabetical - order with matching indentation, but don't freak out too much if the - source isn't consistent. + Add a new file name new_tor_fname to this chunk. Try to insert it in + alphabetical order with matching indentation, but don't freak out too + much if the source isn't consistent. Assumes that this chunk is of the form: FOOBAR = \ X \ Y \ Z + + This function operates on paths relative to the top-level tor + directory. """ prespace = "\t" postspace = "\t\t" @@ -131,20 +179,21 @@ class AutomakeChunk: m = re.match(r'(\s+)(\S+)(\s+)\\', line) if not m: continue - prespace, fname, postspace = m.groups() - if fname > member: - self.insert_before(lineno, member, prespace, postspace) + prespace, cur_tor_fname, postspace = m.groups() + if cur_tor_fname > new_tor_fname: + self.insert_before(lineno, new_tor_fname, prespace, postspace) return - self.insert_at_end(member, prespace, postspace) + self.insert_at_end(new_tor_fname, prespace, postspace) - def insert_before(self, lineno, member, prespace, postspace): + def insert_before(self, lineno, new_tor_fname, prespace, postspace): self.lines.insert(lineno, - "{}{}{}\\\n".format(prespace, member, postspace)) + "{}{}{}\\\n".format(prespace, new_tor_fname, + postspace)) - def insert_at_end(self, member, prespace, postspace): - lastline = self.lines[-1] - self.lines[-1] += '{}\\\n'.format(postspace) - self.lines.append("{}{}\n".format(prespace, member)) + def insert_at_end(self, new_tor_fname, prespace, postspace): + lastline = self.lines[-1].strip() + self.lines[-1] = '{}{}{}\\\n'.format(prespace, lastline, postspace) + self.lines.append("{}{}\n".format(prespace, new_tor_fname)) def dump(self, f): """Write all the lines in this chunk to the file 'f'.""" @@ -153,9 +202,14 @@ class AutomakeChunk: if not line.endswith("\n"): f.write("\n") + if self.hasBlank: + f.write("\n") + class ParsedAutomake: """A sort-of-parsed automake file, with identified chunks into which headers and c files can be inserted. + + This class operates on paths relative to the top-level tor directory. """ def __init__(self): self.chunks = [] @@ -166,12 +220,15 @@ class ParsedAutomake: self.chunks.append(chunk) self.by_type[chunk.kind.lower()] = chunk - def add_file(self, fname, kind): - """Insert a file of kind 'kind' to the appropriate section of this - file. Return True if we added it. + def add_file(self, tor_fname, kind): + """Insert a file tor_fname of kind 'kind' to the appropriate + section of this file. Return True if we added it. + + This function operates on paths relative to the top-level tor + directory. """ if kind.lower() in self.by_type: - self.by_type[kind.lower()].insertMember(fname) + self.by_type[kind.lower()].insertMember(tor_fname) return True else: return False @@ -181,52 +238,77 @@ class ParsedAutomake: for chunk in self.chunks: chunk.dump(f) -def get_include_am_location(fname): - """Find the right include.am file for introducing a new file. Return None - if we can't guess one. +def get_include_am_location(tor_fname): + """Find the right include.am file for introducing a new file + tor_fname. Return None if we can't guess one. Note that this function is imperfect because our include.am layout is not (yet) consistent. + + This function operates on paths relative to the top-level tor directory. """ - td = topdir_file(fname) - m = re.match(r'^lib/([a-z0-9_]*)/', td) + # Strip src for pattern matching, but add it back when returning the path + src_fname = srcdir_file(tor_fname) + m = re.match(r'^(lib|core|feature|app)/([a-z0-9_]*)/', src_fname) if m: - return "src/lib/{}/include.am".format(m.group(1)) - - if re.match(r'^(core|feature|app)/', td): - return "src/core/include.am" + return "src/{}/{}/include.am".format(m.group(1),m.group(2)) - if re.match(r'^test/', td): + if re.match(r'^test/', src_fname): return "src/test/include.am" return None -def run(fn): - """ - Create a new C file and H file corresponding to the filename "fn", and - add them to include.am. +def run(fname): """ + Create a new C file and H file corresponding to the filename "fname", + and add them to the corresponding include.am. - cf = makeext(fn, "c") - hf = makeext(fn, "h") + This function operates on paths relative to the top-level tor directory. + """ - if os.path.exists(cf): - print("{} already exists".format(cf)) + # Make sure we're in the top-level tor directory, + # which contains the src directory + if not os.path.isdir("src"): + raise RuntimeError("Could not find './src/'. " + "Run this script from the top-level tor source " + "directory.") + + # And it looks like a tor/src directory + if not os.path.isfile("src/include.am"): + raise RuntimeError("Could not find './src/include.am'. " + "Run this script from the top-level tor source " + "directory.") + + # Make the file name relative to the top-level tor directory + tor_fname = tordir_file(fname) + # And check that we're adding files to the "src" directory, + # with canonical paths + if tor_fname[:4] != "src/": + raise ValueError("Requested file path '{}' canonicalized to '{}', " + "but the canonical path did not start with 'src/'. " + "Please add files to the src directory." + .format(fname, tor_fname)) + + c_tor_fname = makeext(tor_fname, "c") + h_tor_fname = makeext(tor_fname, "h") + + if os.path.exists(c_tor_fname): + print("{} already exists".format(c_tor_fname)) return 1 - if os.path.exists(hf): - print("{} already exists".format(hf)) + if os.path.exists(h_tor_fname): + print("{} already exists".format(h_tor_fname)) return 1 - with open(cf, 'w') as f: - f.write(instantiate_template(C_FILE_TEMPLATE, cf)) + with open(c_tor_fname, 'w') as f: + f.write(instantiate_template(C_FILE_TEMPLATE, c_tor_fname)) - with open(hf, 'w') as f: - f.write(instantiate_template(HEADER_TEMPLATE, hf)) + with open(h_tor_fname, 'w') as f: + f.write(instantiate_template(HEADER_TEMPLATE, h_tor_fname)) - iam = get_include_am_location(cf) + iam = get_include_am_location(c_tor_fname) if iam is None or not os.path.exists(iam): print("Made files successfully but couldn't identify include.am for {}" - .format(cf)) + .format(c_tor_fname)) return 1 amfile = ParsedAutomake() @@ -238,8 +320,8 @@ def run(fn): cur_chunk = AutomakeChunk() amfile.addChunk(cur_chunk) - amfile.add_file(cf, "sources") - amfile.add_file(hf, "headers") + amfile.add_file(c_tor_fname, "sources") + amfile.add_file(h_tor_fname, "headers") with open(iam+".tmp", 'w') as f: amfile.dump(f) diff --git a/scripts/maint/annotate_ifdef_directives.py b/scripts/maint/annotate_ifdef_directives.py index 102128bfa0..cd70b55c8c 100755 --- a/scripts/maint/annotate_ifdef_directives.py +++ b/scripts/maint/annotate_ifdef_directives.py @@ -57,6 +57,11 @@ Note that only #else and #endif lines are annotated. Existing comments on those lines are removed. """ +# Future imports for Python 2.7, mandatory in 3.0 +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + import re # Any block with fewer than this many lines does not need annotations. diff --git a/scripts/maint/checkIncludes.py b/scripts/maint/checkIncludes.py index 926b201b35..2ca46347f0 100755 --- a/scripts/maint/checkIncludes.py +++ b/scripts/maint/checkIncludes.py @@ -5,6 +5,11 @@ # functionality. This is a stub file that exists so that older git # hooks will know where to look. +# Future imports for Python 2.7, mandatory in 3.0 +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + import sys, os dirname = os.path.split(sys.argv[0])[0] diff --git a/scripts/maint/checkSpace.pl b/scripts/maint/checkSpace.pl index 9c9b68ff9d..f4e6f733c8 100755 --- a/scripts/maint/checkSpace.pl +++ b/scripts/maint/checkSpace.pl @@ -4,9 +4,16 @@ use strict; use warnings; my $found = 0; +my $COLON_POS = 10; + sub msg { $found = 1; - print "$_[0]"; + my $v = shift; + $v =~ /^\s*([^:]+):(.*)$/; + chomp(my $errtype = $1); + my $rest = $2; + my $padding = ' ' x ($COLON_POS - length $errtype); + print "$padding$errtype:$rest\n"; } my $C = 0; @@ -29,7 +36,7 @@ for my $fn (@ARGV) { my $basename = $fn; $basename =~ s#.*/##; if ($basenames{$basename}) { - msg "Duplicate fnames: $fn and $basenames{$basename}.\n"; + msg "dup fname:$fn (same as $basenames{$basename}).\n"; } else { $basenames{$basename} = $fn; } @@ -42,12 +49,12 @@ for my $fn (@ARGV) { # (We insist on lines that end with a single LF character, not # CR LF.) if (/\r/) { - msg " CR:$fn:$.\n"; + msg "CR:$fn:$.\n"; } ## Warn about tabs. # (We only use spaces) if (/\t/) { - msg " TAB:$fn:$.\n"; + msg "TAB:$fn:$.\n"; } ## Warn about labels that don't have a space in front of them # (We indent every label at least one space) @@ -63,12 +70,12 @@ for my $fn (@ARGV) { ## Warn about control keywords without following space. # (We put a space after every 'if', 'while', 'for', 'switch', etc) if ($C && /\s(?:if|while|for|switch)\(/) { - msg " KW(:$fn:$.\n"; + msg "KW(:$fn:$.\n"; } ## Warn about #else #if instead of #elif. # (We only allow #elif) if (($lastline =~ /^\# *else/) and ($_ =~ /^\# *if/)) { - msg " #else#if:$fn:$.\n"; + msg "#else#if:$fn:$.\n"; } ## Warn about some K&R violations # (We use K&R-style C, where open braces go on the same line as @@ -83,19 +90,19 @@ for my $fn (@ARGV) { msg "non-K&R {:$fn:$.\n"; } if (/^\s*else/ and $lastline =~ /\}$/) { - msg " }\\nelse:$fn:$.\n"; + msg "}\\nelse:$fn:$.\n"; } $lastline = $_; ## Warn about unnecessary empty lines. # (Don't put an empty line before a line that contains nothing # but a closing brace.) if ($lastnil && /^\s*}\n/) { - msg " UnnecNL:$fn:$.\n"; + msg "UnnecNL:$fn:$.\n"; } ## Warn about multiple empty lines. # (At most one blank line in a row.) if ($lastnil && /^$/) { - msg " DoubleNL:$fn:$.\n"; + msg "DoubleNL:$fn:$.\n"; } elsif (/^$/) { $lastnil = 1; } else { @@ -105,7 +112,7 @@ for my $fn (@ARGV) { ## accept double-line lines. # (Don't make lines wider than 80 characters, including newline.) if (/^.{80}/) { - msg " Wide:$fn:$.\n"; + msg "Wide:$fn:$.\n"; } ### Juju to skip over comments and strings, since the tests ### we're about to do are okay there. @@ -144,29 +151,28 @@ for my $fn (@ARGV) { } s!"(?:[^\"]+|\\.)*"!"X"!g; next if /^\#/; - ## Warn about C++-style comments. - # (Use C style comments only.) + ## Skip C++-style comments. if (m!//!) { - # msg " //:$fn:$.\n"; + # msg "//:$fn:$.\n"; s!//.*!!; } ## Warn about unquoted braces preceded by non-space. # (No character except a space should come before a {) if (/([^\s'])\{/) { - msg " $1\{:$fn:$.\n"; + msg "$1\{:$fn:$.\n"; } ## Warn about double semi-colons at the end of a line. if (/;;$/) { - msg " double semi-colons at the end of $. in $fn\n" + msg ";;:$fn:$.\n" } ## Warn about multiple internal spaces. #if (/[^\s,:]\s{2,}[^\s\\=]/) { - # msg " X X:$fn:$.\n"; + # msg "X X:$fn:$.\n"; #} ## Warn about { with stuff after. #s/\s+$//; #if (/\{[^\}\\]+$/) { - # msg " {X:$fn:$.\n"; + # msg "{X:$fn:$.\n"; #} ## Warn about function calls with space before parens. # (Don't put a space between the name of a function and its @@ -178,7 +184,7 @@ for my $fn (@ARGV) { $1 ne "void" and $1 ne "__attribute__" and $1 ne "op" and $1 ne "size_t" and $1 ne "double" and $1 ne "uint64_t" and $1 ne "workqueue_reply_t" and $1 ne "bool") { - msg " fn ():$fn:$.\n"; + msg "fn ():$fn:$.\n"; } } ## Warn about functions not declared at start of line. @@ -207,28 +213,28 @@ for my $fn (@ARGV) { ## Check for forbidden functions except when they are # explicitly permitted if (/\bassert\(/ && not /assert OK/) { - msg "assert :$fn:$. (use tor_assert)\n"; + msg "assert:$fn:$. (use tor_assert)\n"; } if (/\bmemcmp\(/ && not /memcmp OK/) { - msg "memcmp :$fn:$. (use {tor,fast}_mem{eq,neq,cmp}\n"; + msg "memcmp:$fn:$. (use {tor,fast}_mem{eq,neq,cmp}\n"; } # always forbidden. if (not /\ OVERRIDE\ /) { if (/\bstrcat\(/ or /\bstrcpy\(/ or /\bsprintf\(/) { - msg "$& :$fn:$.\n"; + msg "$&:$fn:$.\n"; } if (/\bmalloc\(/ or /\bfree\(/ or /\brealloc\(/ or /\bstrdup\(/ or /\bstrndup\(/ or /\bcalloc\(/) { - msg "$& :$fn:$. (use tor_malloc, tor_free, etc)\n"; + msg "$&:$fn:$. (use tor_malloc, tor_free, etc)\n"; } } } } if ($isheader && $C) { if ($seenguard < 2) { - msg "$fn:No #ifndef/#define header guard pair found.\n"; + msg "noguard:$fn (No #ifndef/#define header guard pair found)\n"; } elsif ($guardnames{$guardname}) { - msg "$fn:Guard macro $guardname also used in $guardnames{$guardname}\n"; + msg "dupguard:$fn (Guard macro $guardname also used in $guardnames{$guardname})\n"; } else { $guardnames{$guardname} = $fn; } diff --git a/scripts/maint/checkSpaceTest.sh b/scripts/maint/checkSpaceTest.sh new file mode 100755 index 0000000000..e1d207a1a8 --- /dev/null +++ b/scripts/maint/checkSpaceTest.sh @@ -0,0 +1,36 @@ +#!/bin/sh +# Copyright 2019, The Tor Project, Inc. +# See LICENSE for licensing information + +# Integration test for checkSpace.pl, which we want to rewrite. + +umask 077 +set -e + +# Skip this test if we're running on Windows; we expect line-ending +# issues in that case. +case "$(uname -s)" in + CYGWIN*) WINDOWS=1;; + MINGW*) WINDOWS=1;; + MSYS*) WINDOWS=1;; + *) WINDOWS=0;; +esac +if test "$WINDOWS" = 1; then + # This magic value tells automake that the test has been skipped. + exit 77 +fi + +# make a safe space for temporary files +DATA_DIR=$(mktemp -d -t tor_checkspace_tests.XXXXXX) +trap 'rm -rf "$DATA_DIR"' 0 + +RECEIVED_FNAME="${DATA_DIR}/got.txt" + +cd "$(dirname "$0")/checkspace_tests" + +# we expect this to give an error code. +../checkSpace.pl -C ./*.[ch] ./*/*.[ch] > "${RECEIVED_FNAME}" && exit 1 + +diff -u expected.txt "${RECEIVED_FNAME}" || exit 1 + +echo "OK" diff --git a/scripts/maint/checkspace_tests/dubious.c b/scripts/maint/checkspace_tests/dubious.c new file mode 100644 index 0000000000..59c5f8e4fe --- /dev/null +++ b/scripts/maint/checkspace_tests/dubious.c @@ -0,0 +1,83 @@ + +// The { coming up should be on its own line. +int +foo(void) { + // There should be a space before (1) + if(1) x += 1; + + // The following empty line is unnecessary. + +} + + +// There should be a newline between void and bar. +void bar(void) +{ + // too wide: + testing("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"); +} + +long +bad_spacing() +{ + // here comes a tab + return 2; + // here comes a label without space: +foo: + ; +} + +// Here comes a CR:
+ +// Trailing space: + +int +non_k_and_r(void) +{ + // non-k&r + if (foo) + { + // double-semi + return 1;; + } + else + { + return 2; + } +} + +// #else #if causes a warning. +#if 1 +#else +#if 2 +#else +#endif +#endif + +// always space before a brace. +foo{ +} + +void +unexpected_space(void) +{ + // This space gives a warning. + foobar (77); +} + +void +bad_function_calls(long) +{ + // These are forbidden: + assert(1); + memcmp("a","b",1); + strcat(foo,x); + strcpy(foo,y); + sprintf(foo,"x"); + malloc(7); + free(p); + realloc(p); + strdup(s); + strndup(s,10); + calloc(a,b); +} diff --git a/scripts/maint/checkspace_tests/dubious.h b/scripts/maint/checkspace_tests/dubious.h new file mode 100644 index 0000000000..744ec33955 --- /dev/null +++ b/scripts/maint/checkspace_tests/dubious.h @@ -0,0 +1,4 @@ + +// no guards. + +int foo(int); diff --git a/scripts/maint/checkspace_tests/expected.txt b/scripts/maint/checkspace_tests/expected.txt new file mode 100644 index 0000000000..935b750ef9 --- /dev/null +++ b/scripts/maint/checkspace_tests/expected.txt @@ -0,0 +1,31 @@ + fn() {:./dubious.c:4 + KW(:./dubious.c:6 + UnnecNL:./dubious.c:10 + DoubleNL:./dubious.c:12 + tp fn():./dubious.c:15 + Wide:./dubious.c:17 + TAB:./dubious.c:24 + nosplabel:./dubious.c:26 + CR:./dubious.c:30 + Space@EOL:./dubious.c:32 + non-K&R {:./dubious.c:39 + ;;:./dubious.c:41 + }\nelse:./dubious.c:43 + #else#if:./dubious.c:52 + o{:./dubious.c:58 + fn() {:./dubious.c:58 + fn ():./dubious.c:65 + assert:./dubious.c:72 (use tor_assert) + memcmp:./dubious.c:73 (use {tor,fast}_mem{eq,neq,cmp} + strcat(:./dubious.c:74 + strcpy(:./dubious.c:75 + sprintf(:./dubious.c:76 + malloc(:./dubious.c:77 (use tor_malloc, tor_free, etc) + free(:./dubious.c:78 (use tor_malloc, tor_free, etc) + realloc(:./dubious.c:79 (use tor_malloc, tor_free, etc) + strdup(:./dubious.c:80 (use tor_malloc, tor_free, etc) + strndup(:./dubious.c:81 (use tor_malloc, tor_free, etc) + calloc(:./dubious.c:82 (use tor_malloc, tor_free, etc) + noguard:./dubious.h (No #ifndef/#define header guard pair found) + dupguard:./same_guard.h (Guard macro GUARD_MACRO_H also used in ./good_guard.h) + dup fname:./subdir/dubious.c (same as ./dubious.c). diff --git a/scripts/maint/checkspace_tests/good_guard.h b/scripts/maint/checkspace_tests/good_guard.h new file mode 100644 index 0000000000..b792912d90 --- /dev/null +++ b/scripts/maint/checkspace_tests/good_guard.h @@ -0,0 +1,6 @@ +#ifndef GUARD_MACRO_H +#define GUARD_MACRO_H + +int bar(void); + +#endif diff --git a/scripts/maint/checkspace_tests/same_guard.h b/scripts/maint/checkspace_tests/same_guard.h new file mode 100644 index 0000000000..b792912d90 --- /dev/null +++ b/scripts/maint/checkspace_tests/same_guard.h @@ -0,0 +1,6 @@ +#ifndef GUARD_MACRO_H +#define GUARD_MACRO_H + +int bar(void); + +#endif diff --git a/scripts/maint/checkspace_tests/subdir/dubious.c b/scripts/maint/checkspace_tests/subdir/dubious.c new file mode 100644 index 0000000000..7f22bf79bf --- /dev/null +++ b/scripts/maint/checkspace_tests/subdir/dubious.c @@ -0,0 +1 @@ +// Nothing wrong with this file, but the name is a duplicate. diff --git a/scripts/maint/format_changelog.py b/scripts/maint/format_changelog.py index 08b2155fa3..32085c3602 100755 --- a/scripts/maint/format_changelog.py +++ b/scripts/maint/format_changelog.py @@ -9,6 +9,11 @@ # To run it, pipe a section of the changelog (starting with "Changes # in Tor 0.x.y.z-alpha" through the script.) +# Future imports for Python 2.7, mandatory in 3.0 +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + import os import re import sys @@ -190,7 +195,7 @@ def body_parser(line): elif re.match(r'^\s+\S', line): return TP_HEADTEXT else: - print "Weird line %r"%line + print("Weird line %r"%line, file=sys.stderr) def clean_head(head): return head @@ -198,7 +203,7 @@ def clean_head(head): def head_score(s): m = re.match(r'^ +o (.*)', s) if not m: - print >>sys.stderr, "Can't score %r"%s + print("Can't score %r"%s, file=sys.stderr) return 99999 lw = m.group(1).lower() if lw.startswith("security") and "feature" not in lw: @@ -286,12 +291,12 @@ class ChangeLog(object): self.curgraf.append(line) else: - assert "This" is "unreachable" + assert "This" is "unreachable" # noqa: F632 def lint_head(self, line, head): m = re.match(r'^ *o ([^\(]+)((?:\([^\)]+\))?):', head) if not m: - print >>sys.stderr, "Weird header format on line %s"%line + print("Weird header format on line %s"%line, file=sys.stderr) def lint_item(self, line, grafs, head_type): pass @@ -306,7 +311,7 @@ class ChangeLog(object): def dumpGraf(self,par,indent1,indent2=-1): if not self.wrapText: for line in par: - print line + print(line) return if indent2 == -1: @@ -320,17 +325,17 @@ class ChangeLog(object): def dumpPreheader(self, graf): self.dumpGraf(graf, 0) - print + print() def dumpMainhead(self, head): - print head + print(head) def dumpHeadGraf(self, graf): self.dumpGraf(graf, 2) - print + print() def dumpSectionHeader(self, header): - print header + print(header) def dumpStartOfSections(self): pass @@ -339,10 +344,10 @@ class ChangeLog(object): pass def dumpEndOfSection(self): - print + print() def dumpEndOfChangelog(self): - print + print() def dumpDrupalBreak(self): pass @@ -350,7 +355,7 @@ class ChangeLog(object): def dumpItem(self, grafs): self.dumpGraf(grafs[0],4,6) for par in grafs[1:]: - print + print() self.dumpGraf(par,6,6) def collateAndSortSections(self): @@ -389,7 +394,7 @@ class ChangeLog(object): self.dumpStartOfSections() for _,head,items in self.sections: if not head.endswith(':'): - print >>sys.stderr, "adding : to %r"%head + print("adding : to %r"%head, file=sys.stderr) head = head + ":" self.dumpSectionHeader(head) for _,grafs in items: @@ -445,16 +450,16 @@ class HTMLChangeLog(ChangeLog): pass def dumpStartOfSections(self): - print "<ul>\n" + print("<ul>\n") def dumpEndOfSections(self): - print "</ul>\n" + print("</ul>\n") def dumpDrupalBreak(self): - print "\n</ul>\n" - print "<p> </p>" - print "\n<!--break-->\n\n" - print "<ul>" + print("\n</ul>\n") + print("<p> </p>") + print("\n<!--break-->\n\n") + print("<ul>") def dumpItem(self, grafs): grafs[0][0] = grafs[0][0].replace(" - ", "", 1).lstrip() @@ -464,7 +469,7 @@ class HTMLChangeLog(ChangeLog): self.htmlPar(par) else: self.htmlText(grafs[0]) - print + print() op = optparse.OptionParser(usage="usage: %prog [options] [filename]") op.add_option('-W', '--no-wrap', action='store_false', @@ -560,7 +565,7 @@ if options.firstOnly: sys.exit(0) if nextline is not None: - print nextline + print(nextline) for line in sys.stdin: sys.stdout.write(line) diff --git a/scripts/maint/lintChanges.py b/scripts/maint/lintChanges.py index 82c118f07e..88a865a572 100755 --- a/scripts/maint/lintChanges.py +++ b/scripts/maint/lintChanges.py @@ -1,7 +1,10 @@ #!/usr/bin/python +# Future imports for Python 2.7, mandatory in 3.0 +from __future__ import division from __future__ import print_function -from __future__ import with_statement +from __future__ import unicode_literals + import sys import re import os diff --git a/scripts/maint/locatemissingdoxygen.py b/scripts/maint/locatemissingdoxygen.py index 797bf8176f..7733977359 100755 --- a/scripts/maint/locatemissingdoxygen.py +++ b/scripts/maint/locatemissingdoxygen.py @@ -7,6 +7,11 @@ to highlight the undocumented stuff. """ +# Future imports for Python 2.7, mandatory in 3.0 +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + import os import re import shutil @@ -29,10 +34,10 @@ def buildWarnings(): def count(fn): if os.path.abspath(fn) not in warnings: - print "0\t%s"%fn + print("0\t%s"%fn) else: n = len(warnings[os.path.abspath(fn)]) - print "%d\t%s"%(n,fn) + print("%d\t%s"%(n,fn)) def getIndentation(line): s = line.lstrip() @@ -62,7 +67,7 @@ def annotate(filename): if __name__ == '__main__': if len(sys.argv) == 1: - print "Usage: locatemissingdoxygen.py [-A] filename... <doxygen_log" + print("Usage: locatemissingdoxygen.py [-A] filename... <doxygen_log") sys.exit(1) buildWarnings() if sys.argv[1] == '-A': diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt index 7b15b37f8c..d89b80c1b3 100644 --- a/scripts/maint/practracker/exceptions.txt +++ b/scripts/maint/practracker/exceptions.txt @@ -6,8 +6,12 @@ # # There are three kinds of problems that we recognize right now: # function-size -- a function of more than 100 lines. -# file-size -- a file of more than 3000 lines. -# include-count -- a file with more than 50 #includes. +# file-size -- a .c file of more than 3000 lines, or a .h +# file with more than 500 lines. +# include-count -- a .c file with more than 50 #includes, +# or a .h file with more than 15 #includes. +# dependency-violation -- a file includes a header that it should +# not, according to an advisory .may_include file. # # Each line below represents a single exception that practracker should # _ignore_. Each line has four parts: @@ -29,34 +33,35 @@ # # Remember: It is better to fix the problem than to add a new exception! -problem file-size /src/app/config/config.c 8518 -problem include-count /src/app/config/config.c 89 -problem function-size /src/app/config/config.c:options_act_reversible() 296 -problem function-size /src/app/config/config.c:options_act() 589 +problem file-size /src/app/config/config.c 7400 +problem include-count /src/app/config/config.c 80 +problem function-size /src/app/config/config.c:options_act_reversible() 298 +problem function-size /src/app/config/config.c:options_act() 381 problem function-size /src/app/config/config.c:resolve_my_address() 190 -problem function-size /src/app/config/config.c:options_validate() 1209 -problem function-size /src/app/config/config.c:options_init_from_torrc() 207 -problem function-size /src/app/config/config.c:options_init_from_string() 171 -problem function-size /src/app/config/config.c:options_init_logs() 145 +problem function-size /src/app/config/config.c:options_validate_cb() 780 +problem function-size /src/app/config/config.c:options_init_from_torrc() 188 +problem function-size /src/app/config/config.c:options_init_from_string() 103 +problem function-size /src/app/config/config.c:options_init_logs() 125 problem function-size /src/app/config/config.c:parse_bridge_line() 104 -problem function-size /src/app/config/config.c:parse_transport_line() 189 +problem function-size /src/app/config/config.c:pt_parse_transport_line() 189 problem function-size /src/app/config/config.c:parse_dir_authority_line() 150 problem function-size /src/app/config/config.c:parse_dir_fallback_line() 101 -problem function-size /src/app/config/config.c:parse_port_config() 446 -problem function-size /src/app/config/config.c:parse_ports() 168 -problem file-size /src/app/config/or_options_st.h 1112 -problem include-count /src/app/main/main.c 68 +problem function-size /src/app/config/config.c:port_parse_config() 450 +problem function-size /src/app/config/config.c:parse_ports() 132 +problem file-size /src/app/config/or_options_st.h 1115 +problem include-count /src/app/main/main.c 69 problem function-size /src/app/main/main.c:dumpstats() 102 -problem function-size /src/app/main/main.c:tor_init() 137 +problem function-size /src/app/main/main.c:tor_init() 101 problem function-size /src/app/main/main.c:sandbox_init_filter() 291 problem function-size /src/app/main/main.c:run_tor_main_loop() 105 problem function-size /src/app/main/ntmain.c:nt_service_install() 126 problem dependency-violation /src/core/crypto/hs_ntor.c 1 +problem dependency-violation /src/core/crypto/hs_ntor.h 1 problem dependency-violation /src/core/crypto/onion_crypto.c 5 problem dependency-violation /src/core/crypto/onion_fast.c 1 problem dependency-violation /src/core/crypto/onion_tap.c 3 problem dependency-violation /src/core/crypto/relay_crypto.c 9 -problem file-size /src/core/mainloop/connection.c 5569 +problem file-size /src/core/mainloop/connection.c 5577 problem include-count /src/core/mainloop/connection.c 62 problem function-size /src/core/mainloop/connection.c:connection_free_minimal() 185 problem function-size /src/core/mainloop/connection.c:connection_listener_new() 324 @@ -68,20 +73,22 @@ problem function-size /src/core/mainloop/connection.c:connection_handle_read_imp problem function-size /src/core/mainloop/connection.c:connection_buf_read_from_socket() 180 problem function-size /src/core/mainloop/connection.c:connection_handle_write_impl() 241 problem function-size /src/core/mainloop/connection.c:assert_connection_ok() 143 -problem dependency-violation /src/core/mainloop/connection.c 44 +problem dependency-violation /src/core/mainloop/connection.c 47 problem dependency-violation /src/core/mainloop/cpuworker.c 12 -problem include-count /src/core/mainloop/mainloop.c 63 +problem include-count /src/core/mainloop/mainloop.c 64 problem function-size /src/core/mainloop/mainloop.c:conn_close_if_marked() 108 problem function-size /src/core/mainloop/mainloop.c:run_connection_housekeeping() 123 -problem dependency-violation /src/core/mainloop/mainloop.c 49 +problem dependency-violation /src/core/mainloop/mainloop.c 50 problem dependency-violation /src/core/mainloop/mainloop_pubsub.c 1 problem dependency-violation /src/core/mainloop/mainloop_sys.c 1 problem dependency-violation /src/core/mainloop/netstatus.c 4 problem dependency-violation /src/core/mainloop/periodic.c 2 problem dependency-violation /src/core/or/address_set.c 1 +problem dependency-violation /src/core/or/cell_queue_st.h 1 problem file-size /src/core/or/channel.c 3487 problem dependency-violation /src/core/or/channel.c 9 -problem file-size /src/core/or/channel.h 780 +problem file-size /src/core/or/channel.h 781 +problem dependency-violation /src/core/or/channel.h 1 problem dependency-violation /src/core/or/channelpadding.c 6 problem function-size /src/core/or/channeltls.c:channel_tls_handle_var_cell() 160 problem function-size /src/core/or/channeltls.c:channel_tls_process_versions_cell() 170 @@ -96,15 +103,16 @@ problem function-size /src/core/or/circuitbuild.c:choose_good_exit_server_genera problem dependency-violation /src/core/or/circuitbuild.c 25 problem include-count /src/core/or/circuitlist.c 55 problem function-size /src/core/or/circuitlist.c:HT_PROTOTYPE() 109 -problem function-size /src/core/or/circuitlist.c:circuit_free_() 143 +problem function-size /src/core/or/circuitlist.c:circuit_free_() 146 problem function-size /src/core/or/circuitlist.c:circuit_find_to_cannibalize() 101 problem function-size /src/core/or/circuitlist.c:circuit_about_to_free() 120 problem function-size /src/core/or/circuitlist.c:circuits_handle_oom() 117 problem dependency-violation /src/core/or/circuitlist.c 19 +problem dependency-violation /src/core/or/circuitlist.h 1 problem function-size /src/core/or/circuitmux.c:circuitmux_set_policy() 109 problem function-size /src/core/or/circuitmux.c:circuitmux_attach_circuit() 113 problem dependency-violation /src/core/or/circuitmux_ewma.c 2 -problem file-size /src/core/or/circuitpadding.c 3096 +problem file-size /src/core/or/circuitpadding.c 3098 problem function-size /src/core/or/circuitpadding.c:circpad_machine_schedule_padding() 113 problem dependency-violation /src/core/or/circuitpadding.c 6 problem file-size /src/core/or/circuitpadding.h 813 @@ -119,33 +127,35 @@ problem function-size /src/core/or/circuituse.c:circuit_expire_building() 394 problem function-size /src/core/or/circuituse.c:circuit_log_ancient_one_hop_circuits() 126 problem function-size /src/core/or/circuituse.c:circuit_build_failed() 149 problem function-size /src/core/or/circuituse.c:circuit_launch_by_extend_info() 108 -problem function-size /src/core/or/circuituse.c:circuit_get_open_circ_or_launch() 352 +problem function-size /src/core/or/circuituse.c:circuit_get_open_circ_or_launch() 351 problem function-size /src/core/or/circuituse.c:connection_ap_handshake_attach_circuit() 244 -problem dependency-violation /src/core/or/circuituse.c 23 +problem dependency-violation /src/core/or/circuituse.c 24 problem function-size /src/core/or/command.c:command_process_create_cell() 156 problem function-size /src/core/or/command.c:command_process_relay_cell() 132 problem dependency-violation /src/core/or/command.c 8 -problem file-size /src/core/or/connection_edge.c 4596 +problem file-size /src/core/or/connection_edge.c 4640 problem include-count /src/core/or/connection_edge.c 65 problem function-size /src/core/or/connection_edge.c:connection_ap_expire_beginning() 117 -problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_rewrite() 191 +problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_rewrite() 193 problem function-size /src/core/or/connection_edge.c:connection_ap_handle_onion() 185 -problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_rewrite_and_attach() 421 +problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_rewrite_and_attach() 420 problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_send_begin() 111 problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_socks_resolved() 101 problem function-size /src/core/or/connection_edge.c:connection_exit_begin_conn() 185 problem function-size /src/core/or/connection_edge.c:connection_exit_connect() 102 problem dependency-violation /src/core/or/connection_edge.c 27 +problem dependency-violation /src/core/or/connection_edge.h 1 problem file-size /src/core/or/connection_or.c 3122 problem include-count /src/core/or/connection_or.c 51 problem function-size /src/core/or/connection_or.c:connection_or_group_set_badness_() 105 problem function-size /src/core/or/connection_or.c:connection_or_client_learned_peer_id() 142 problem function-size /src/core/or/connection_or.c:connection_or_compute_authenticate_cell_body() 231 problem dependency-violation /src/core/or/connection_or.c 20 -problem dependency-violation /src/core/or/dos.c 5 +problem dependency-violation /src/core/or/dos.c 6 problem dependency-violation /src/core/or/onion.c 2 problem file-size /src/core/or/or.h 1107 problem include-count /src/core/or/or.h 49 +problem dependency-violation /src/core/or/or.h 1 problem dependency-violation /src/core/or/or_periodic.c 1 problem file-size /src/core/or/policies.c 3249 problem function-size /src/core/or/policies.c:policy_summarize() 107 @@ -179,7 +189,7 @@ problem function-size /src/feature/client/addressmap.c:addressmap_rewrite() 109 problem function-size /src/feature/client/bridges.c:rewrite_node_address_for_bridge() 126 problem function-size /src/feature/client/circpathbias.c:pathbias_measure_close_rate() 108 problem function-size /src/feature/client/dnsserv.c:evdns_server_callback() 153 -problem file-size /src/feature/client/entrynodes.c 3824 +problem file-size /src/feature/client/entrynodes.c 3825 problem function-size /src/feature/client/entrynodes.c:entry_guards_upgrade_waiting_circuits() 155 problem function-size /src/feature/client/entrynodes.c:entry_guard_parse_from_state() 246 problem file-size /src/feature/client/entrynodes.h 639 @@ -190,11 +200,11 @@ problem function-size /src/feature/control/control.c:connection_control_process_ problem function-size /src/feature/control/control_auth.c:handle_control_authenticate() 186 problem function-size /src/feature/control/control_cmd.c:handle_control_extendcircuit() 150 problem function-size /src/feature/control/control_cmd.c:handle_control_add_onion() 256 -problem function-size /src/feature/control/control_cmd.c:add_onion_helper_keyarg() 116 -problem function-size /src/feature/control/control_events.c:control_event_stream_status() 118 +problem function-size /src/feature/control/control_cmd.c:add_onion_helper_keyarg() 118 +problem function-size /src/feature/control/control_events.c:control_event_stream_status() 124 problem include-count /src/feature/control/control_getinfo.c 54 problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_misc() 108 -problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_dir() 302 +problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_dir() 297 problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_events() 234 problem function-size /src/feature/dirauth/bwauth.c:dirserv_read_measured_bandwidths() 121 problem file-size /src/feature/dirauth/dirvote.c 4700 @@ -215,8 +225,8 @@ problem function-size /src/feature/dircache/consdiffmgr.c:consdiffmgr_cleanup() problem function-size /src/feature/dircache/consdiffmgr.c:consdiffmgr_rescan_flavor_() 111 problem function-size /src/feature/dircache/consdiffmgr.c:consensus_diff_worker_threadfn() 132 problem function-size /src/feature/dircache/dircache.c:handle_get_current_consensus() 165 -problem function-size /src/feature/dircache/dircache.c:directory_handle_command_post() 119 -problem file-size /src/feature/dirclient/dirclient.c 3215 +problem function-size /src/feature/dircache/dircache.c:directory_handle_command_post() 124 +problem file-size /src/feature/dirclient/dirclient.c 3165 problem include-count /src/feature/dirclient/dirclient.c 51 problem function-size /src/feature/dirclient/dirclient.c:directory_get_from_dirserver() 126 problem function-size /src/feature/dirclient/dirclient.c:directory_initiate_request() 201 @@ -246,8 +256,8 @@ problem function-size /src/feature/hs/hs_descriptor.c:desc_encode_v3() 101 problem function-size /src/feature/hs/hs_descriptor.c:decrypt_desc_layer() 111 problem function-size /src/feature/hs/hs_descriptor.c:decode_introduction_point() 122 problem function-size /src/feature/hs/hs_descriptor.c:desc_decode_superencrypted_v3() 107 -problem function-size /src/feature/hs/hs_descriptor.c:desc_decode_encrypted_v3() 107 -problem file-size /src/feature/hs/hs_service.c 4116 +problem function-size /src/feature/hs/hs_descriptor.c:desc_decode_encrypted_v3() 109 +problem file-size /src/feature/hs/hs_service.c 4172 problem function-size /src/feature/keymgt/loadkey.c:ed_key_init_from_file() 326 problem function-size /src/feature/nodelist/authcert.c:trusted_dirs_load_certs_from_string() 123 problem function-size /src/feature/nodelist/authcert.c:authority_certs_fetch_missing() 295 @@ -260,7 +270,7 @@ problem function-size /src/feature/nodelist/node_select.c:router_pick_directory_ problem function-size /src/feature/nodelist/node_select.c:compute_weighted_bandwidths() 203 problem function-size /src/feature/nodelist/node_select.c:router_pick_trusteddirserver_impl() 112 problem function-size /src/feature/nodelist/nodelist.c:compute_frac_paths_available() 190 -problem file-size /src/feature/nodelist/routerlist.c 3241 +problem file-size /src/feature/nodelist/routerlist.c 3239 problem function-size /src/feature/nodelist/routerlist.c:router_rebuild_store() 148 problem function-size /src/feature/nodelist/routerlist.c:router_add_to_routerlist() 168 problem function-size /src/feature/nodelist/routerlist.c:routerlist_remove_old_routers() 121 @@ -269,8 +279,8 @@ problem function-size /src/feature/nodelist/routerlist.c:update_extrainfo_downlo problem function-size /src/feature/relay/dns.c:dns_resolve_impl() 131 problem function-size /src/feature/relay/dns.c:configure_nameservers() 161 problem function-size /src/feature/relay/dns.c:evdns_callback() 108 -problem file-size /src/feature/relay/router.c 3522 -problem include-count /src/feature/relay/router.c 56 +problem file-size /src/feature/relay/router.c 3520 +problem include-count /src/feature/relay/router.c 57 problem function-size /src/feature/relay/router.c:init_keys() 252 problem function-size /src/feature/relay/router.c:get_my_declared_family() 114 problem function-size /src/feature/relay/router.c:router_build_fresh_unsigned_routerinfo() 136 @@ -279,14 +289,14 @@ problem function-size /src/feature/relay/routerkeys.c:load_ed_keys() 294 problem function-size /src/feature/rend/rendcache.c:rend_cache_store_v2_desc_as_client() 190 problem function-size /src/feature/rend/rendclient.c:rend_client_send_introduction() 219 problem function-size /src/feature/rend/rendcommon.c:rend_encode_v2_descriptors() 221 -problem function-size /src/feature/rend/rendmid.c:rend_mid_establish_intro_legacy() 104 +problem function-size /src/feature/rend/rendmid.c:rend_mid_establish_intro_legacy() 105 problem function-size /src/feature/rend/rendparse.c:rend_parse_v2_service_descriptor() 181 problem function-size /src/feature/rend/rendparse.c:rend_parse_introduction_points() 129 -problem file-size /src/feature/rend/rendservice.c 4511 +problem file-size /src/feature/rend/rendservice.c 4522 problem function-size /src/feature/rend/rendservice.c:rend_service_prune_list_impl_() 107 problem function-size /src/feature/rend/rendservice.c:rend_config_service() 162 problem function-size /src/feature/rend/rendservice.c:rend_service_load_auth_keys() 178 -problem function-size /src/feature/rend/rendservice.c:rend_service_receive_introduction() 330 +problem function-size /src/feature/rend/rendservice.c:rend_service_receive_introduction() 334 problem function-size /src/feature/rend/rendservice.c:rend_service_parse_intro_for_v3() 111 problem function-size /src/feature/rend/rendservice.c:rend_service_decrypt_intro() 112 problem function-size /src/feature/rend/rendservice.c:rend_service_intro_has_opened() 126 @@ -310,7 +320,7 @@ problem function-size /src/lib/net/address.c:tor_addr_compare_masked() 110 problem function-size /src/lib/net/inaddr.c:tor_inet_pton() 107 problem function-size /src/lib/net/socketpair.c:tor_ersatz_socketpair() 102 problem function-size /src/lib/osinfo/uname.c:get_uname() 116 -problem function-size /src/lib/process/process_unix.c:process_unix_exec() 220 +problem function-size /src/lib/process/process_unix.c:process_unix_exec() 213 problem function-size /src/lib/process/process_win32.c:process_win32_exec() 151 problem function-size /src/lib/process/process_win32.c:process_win32_create_pipe() 109 problem function-size /src/lib/process/restrict.c:set_max_file_descriptors() 102 @@ -324,12 +334,3 @@ problem function-size /src/tools/tor-gencert.c:parse_commandline() 111 problem function-size /src/tools/tor-resolve.c:build_socks5_resolve_request() 102 problem function-size /src/tools/tor-resolve.c:do_resolve() 171 problem function-size /src/tools/tor-resolve.c:main() 112 - -problem dependency-violation /scripts/maint/practracker/testdata/a.c 3 -problem dependency-violation /scripts/maint/practracker/testdata/header.h 3 -problem dependency-violation /src/core/crypto/hs_ntor.h 1 -problem dependency-violation /src/core/or/cell_queue_st.h 1 -problem dependency-violation /src/core/or/channel.h 1 -problem dependency-violation /src/core/or/circuitlist.h 1 -problem dependency-violation /src/core/or/connection_edge.h 1 -problem dependency-violation /src/core/or/or.h 1 diff --git a/scripts/maint/practracker/includes.py b/scripts/maint/practracker/includes.py index 397439b4ef..fe0f32e253 100755 --- a/scripts/maint/practracker/includes.py +++ b/scripts/maint/practracker/includes.py @@ -13,10 +13,16 @@ file in each directory. This file contains empty lines, #-prefixed comments, filenames (like "lib/foo/bar.h") and file globs (like lib/*/*.h) for files that are permitted. -""" + The script exits with an error if any non-permitted includes are found. + .may_include files that contain "!advisory" are considered advisory. + Advisory .may_include files only result in warnings, rather than errors. +""" +# Future imports for Python 2.7, mandatory in 3.0 +from __future__ import division from __future__ import print_function +from __future__ import unicode_literals import fnmatch import os @@ -36,7 +42,11 @@ def warn(msg): def fname_is_c(fname): """ Return true iff 'fname' is the name of a file that we should search for possibly disallowed #include directives. """ - return fname.endswith(".h") or fname.endswith(".c") + if fname.endswith(".h") or fname.endswith(".c"): + bname = os.path.basename(fname) + return not (bname.startswith(".") or bname.startswith("#")) + else: + return False INCLUDE_PATTERN = re.compile(r'\s*#\s*include\s+"([^"]*)"') RULES_FNAME = ".may_include" @@ -235,7 +245,7 @@ def run_check_includes(topdir, list_unused=False, log_sorted_levels=False, trouble = True if trouble: - err( + warn( """To change which includes are allowed in a C file, edit the {} files in its enclosing directory.""".format(RULES_FNAME)) sys.exit(1) diff --git a/scripts/maint/practracker/metrics.py b/scripts/maint/practracker/metrics.py index 4c62bc2425..ae88b84f31 100644 --- a/scripts/maint/practracker/metrics.py +++ b/scripts/maint/practracker/metrics.py @@ -4,6 +4,11 @@ # These are currently ad-hoc string operations and regexps. # We might want to use a proper static analysis library in the future, if we want to get more advanced metrics. +# Future imports for Python 2.7, mandatory in 3.0 +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + import re def get_file_len(f): diff --git a/scripts/maint/practracker/practracker.py b/scripts/maint/practracker/practracker.py index 71741265f6..79b13cb056 100755 --- a/scripts/maint/practracker/practracker.py +++ b/scripts/maint/practracker/practracker.py @@ -19,14 +19,18 @@ problems in the Tor source, use the --regen flag: $ python3 --regen ./scripts/maint/practracker/practracker.py . """ +# Future imports for Python 2.7, mandatory in 3.0 +from __future__ import division from __future__ import print_function +from __future__ import unicode_literals -import os, sys +import codecs, os, sys import metrics import util import problem import includes +import shutil # The filename of the exceptions file (it should be placed in the practracker directory) EXCEPTIONS_FNAME = "./exceptions.txt" @@ -59,12 +63,8 @@ TOR_TOPDIR = None ####################################################### -if sys.version_info[0] <= 2: - def open_file(fname): - return open(fname, 'r') -else: - def open_file(fname): - return open(fname, 'r', encoding='utf-8') +def open_file(fname): + return codecs.open(fname, 'r', encoding='utf-8') def consider_file_size(fname, f): """Consider the size of 'f' and yield an FileSizeItem for it. @@ -147,7 +147,7 @@ HEADER="""\ # file-size -- a .c file of more than {MAX_FILE_SIZE} lines, or a .h # file with more than {MAX_H_FILE_SIZE} lines. # include-count -- a .c file with more than {MAX_INCLUDE_COUNT} #includes, - or a .h file with more than {MAX_H_INCLUDE_COUNT} #includes. +# or a .h file with more than {MAX_H_INCLUDE_COUNT} #includes. # dependency-violation -- a file includes a header that it should # not, according to an advisory .may_include file. # @@ -182,6 +182,9 @@ def main(argv): help="Regenerate the exceptions file") parser.add_argument("--list-overbroad", action="store_true", help="List over-broad exceptions") + parser.add_argument("--regen-overbroad", action="store_true", + help="Regenerate the exceptions file, " + "removing over-broad exceptions.") parser.add_argument("--exceptions", help="Override the location for the exceptions file") parser.add_argument("--strict", action="store_true", @@ -224,8 +227,9 @@ def main(argv): filt.addThreshold(problem.DependencyViolationItem("*.c", int(args.max_dependency_violations))) filt.addThreshold(problem.DependencyViolationItem("*.h", int(args.max_dependency_violations))) - if args.list_overbroad and args.regen: - print("Cannot use --regen with --list-overbroad", + if args.list_overbroad + args.regen + args.regen_overbroad > 1: + print("Cannot use more than one of --regen, --list-overbroad, and " + "--regen-overbroad.", file=sys.stderr) sys.exit(1) @@ -244,13 +248,15 @@ def main(argv): ProblemVault = problem.ProblemVault(exceptions_file) problem_file = sys.stdout - if args.list_overbroad: - # If we're listing overbroad exceptions, don't list problems. + if args.list_overbroad or args.regen_overbroad: + # If we're looking for overbroad exceptions, don't list problems + # immediately to the problem file. problem_file = util.NullFile() # 2.1) Adjust the exceptions so that we warn only about small problems, # and produce errors on big ones. - if not (args.regen or args.list_overbroad or args.strict): + if not (args.regen or args.list_overbroad or args.regen_overbroad or + args.strict): ProblemVault.set_tolerances(TOLERANCE_FNS) # 3) Go through all the files and report problems if they are not exceptions @@ -266,7 +272,17 @@ def main(argv): if args.regen: tmpfile.close() - os.rename(tmpname, exceptions_file) + shutil.move(tmpname, exceptions_file) + sys.exit(0) + + if args.regen_overbroad: + tmpname = exceptions_file + ".tmp" + tmpfile = open(tmpname, "w") + tmpfile.write(HEADER) + for item in ProblemVault.list_exceptions_without_overbroad(): + print(item, file=tmpfile) + tmpfile.close() + shutil.move(tmpname, exceptions_file) sys.exit(0) # If new issues were found, try to give out some advice to the developer on how to resolve it. @@ -293,6 +309,7 @@ variable. else: print(ex, "->", p.metric_value) + sys.exit(found_new_issues) if __name__ == '__main__': diff --git a/scripts/maint/practracker/practracker_tests.py b/scripts/maint/practracker/practracker_tests.py index 45719d6cb7..8d0418880c 100755 --- a/scripts/maint/practracker/practracker_tests.py +++ b/scripts/maint/practracker/practracker_tests.py @@ -2,6 +2,11 @@ """Some simple tests for practracker metrics""" +# Future imports for Python 2.7, mandatory in 3.0 +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + import unittest try: diff --git a/scripts/maint/practracker/problem.py b/scripts/maint/practracker/problem.py index d21840a213..a3255dcc80 100644 --- a/scripts/maint/practracker/problem.py +++ b/scripts/maint/practracker/problem.py @@ -7,7 +7,10 @@ problem is worse than a registered exception so that it only warns when things get worse. """ +# Future imports for Python 2.7, mandatory in 3.0 +from __future__ import division from __future__ import print_function +from __future__ import unicode_literals import os.path import re @@ -26,6 +29,8 @@ class ProblemVault(object): def __init__(self, exception_fname=None): # Exception dictionary: { problem.key() : Problem object } self.exceptions = {} + # Exception list: list of Problem objects, in the order added. + self.exception_list = [] # Exception dictionary: maps key to the problem it was used to # suppress. self.used_exception_for = {} @@ -60,6 +65,7 @@ class ProblemVault(object): sys.exit(1) self.exceptions[problem.key()] = problem + self.exception_list.append(problem) #print "Registering exception: %s" % problem def register_problem(self, problem): @@ -95,6 +101,24 @@ class ProblemVault(object): if p is None or e.is_worse_than(p): yield (e, p) + def list_exceptions_without_overbroad(self): + """Return an iterator of new problems, such that overbroad + exceptions are replaced with minimally broad versions, or removed. + """ + for e in self.exception_list: + p = self.used_exception_for.get(e.key()) + if p is None: + # This exception wasn't needed at all. + continue + if e.is_worse_than(p): + # The exception is worse than the problem we found. + # Yield the problem as the new exception value. + yield p + else: + # The problem is as bad as the exception, or worse. + # Yield the exception. + yield e + def set_tolerances(self, fns): """Adjust the tolerances for the exceptions in this vault. Takes a map of problem type to a function that adjusts the permitted diff --git a/scripts/maint/practracker/test_practracker.sh b/scripts/maint/practracker/test_practracker.sh index 207a5ceded..e29b9106de 100755 --- a/scripts/maint/practracker/test_practracker.sh +++ b/scripts/maint/practracker/test_practracker.sh @@ -1,5 +1,8 @@ #!/bin/sh +# Fail if any subprocess fails unexpectedly +set -e + umask 077 unset TOR_DISABLE_PRACTRACKER @@ -16,6 +19,11 @@ if test "${PRACTRACKER_DIR}" = "" || PRACTRACKER_DIR=$(dirname "$0") fi +# Change to the tor directory, and canonicalise PRACTRACKER_DIR, +# so paths in practracker output are consistent, even in out-of-tree builds +cd "${PRACTRACKER_DIR}"/../../.. +PRACTRACKER_DIR="scripts/maint/practracker" + TMPDIR="$(mktemp -d -t pracktracker.test.XXXXXX)" if test -z "${TMPDIR}" || test ! -d "${TMPDIR}" ; then echo >&2 "mktemp failed." @@ -33,11 +41,11 @@ run_practracker() { --max-h-include-count=0 \ --max-include-count=0 \ --terse \ - "${DATA}/" "$@"; + "${DATA}/" "$@" || echo "practracker exit status: $?" } compare() { # we can't use cmp because we need to use -b for windows - diff -b -u "$@" > "${TMPDIR}/test-diff" + diff -b -u "$@" > "${TMPDIR}/test-diff" || true if test -z "$(cat "${TMPDIR}"/test-diff)"; then echo "OK" else @@ -49,22 +57,40 @@ compare() { echo "unit tests:" -"${PYTHON:-python}" "${PRACTRACKER_DIR}/practracker_tests.py" || exit 1 +"${PYTHON:-python}" "${PRACTRACKER_DIR}/practracker_tests.py" echo "ex0:" -run_practracker --exceptions "${DATA}/ex0.txt" > "${TMPDIR}/ex0-received.txt" +run_practracker --exceptions "${DATA}/ex0.txt" \ + > "${TMPDIR}/ex0-received.txt" 2>&1 -compare "${TMPDIR}/ex0-received.txt" "${DATA}/ex0-expected.txt" +compare "${TMPDIR}/ex0-received.txt" \ + "${DATA}/ex0-expected.txt" echo "ex1:" -run_practracker --exceptions "${DATA}/ex1.txt" > "${TMPDIR}/ex1-received.txt" +run_practracker --exceptions "${DATA}/ex1.txt" \ + > "${TMPDIR}/ex1-received.txt" 2>&1 -compare "${TMPDIR}/ex1-received.txt" "${DATA}/ex1-expected.txt" +compare "${TMPDIR}/ex1-received.txt" \ + "${DATA}/ex1-expected.txt" echo "ex1.overbroad:" -run_practracker --exceptions "${DATA}/ex1.txt" --list-overbroad > "${TMPDIR}/ex1-overbroad-received.txt" +run_practracker --exceptions "${DATA}/ex1.txt" --list-overbroad \ + > "${TMPDIR}/ex1-overbroad-received.txt" 2>&1 + +compare "${TMPDIR}/ex1-overbroad-received.txt" \ + "${DATA}/ex1-overbroad-expected.txt" + +echo "ex1.regen:" + +cp "${DATA}/ex1.txt" "${TMPDIR}/ex1-copy.txt" +run_practracker --exceptions "${TMPDIR}/ex1-copy.txt" --regen >/dev/null 2>&1 +compare "${TMPDIR}/ex1-copy.txt" "${DATA}/ex1-regen-expected.txt" + +echo "ex1.regen_overbroad:" -compare "${TMPDIR}/ex1-overbroad-received.txt" "${DATA}/ex1-overbroad-expected.txt" +cp "${DATA}/ex1.txt" "${TMPDIR}/ex1-copy.txt" +run_practracker --exceptions "${TMPDIR}/ex1-copy.txt" --regen-overbroad >/dev/null 2>&1 +compare "${TMPDIR}/ex1-copy.txt" "${DATA}/ex1-regen-overbroad-expected.txt" diff --git a/scripts/maint/practracker/testdata/.may_include b/scripts/maint/practracker/testdata/.may_include index 40bf8155d9..8542a35807 100644 --- a/scripts/maint/practracker/testdata/.may_include +++ b/scripts/maint/practracker/testdata/.may_include @@ -1,3 +1,4 @@ !advisory permitted.h +ext/good.c diff --git a/scripts/maint/practracker/testdata/a.c b/scripts/maint/practracker/testdata/a.c index 1939773f57..3c338ab40d 100644 --- a/scripts/maint/practracker/testdata/a.c +++ b/scripts/maint/practracker/testdata/a.c @@ -5,6 +5,9 @@ # include "permitted.h" +#include "ext/good.c" +#include "bad.c" + int i_am_a_function(void) { diff --git a/scripts/maint/practracker/testdata/ex0-expected.txt b/scripts/maint/practracker/testdata/ex0-expected.txt index 5f3d9e5aec..c9fb83bac3 100644 --- a/scripts/maint/practracker/testdata/ex0-expected.txt +++ b/scripts/maint/practracker/testdata/ex0-expected.txt @@ -1,11 +1,13 @@ -problem file-size a.c 38 -problem include-count a.c 4 +Unusual pattern permitted.h in scripts/maint/practracker/testdata +problem file-size a.c 41 +problem include-count a.c 6 problem function-size a.c:i_am_a_function() 9 problem function-size a.c:another_function() 12 -problem dependency-violation a.c 3 +problem dependency-violation a.c 4 problem file-size b.c 15 problem function-size b.c:foo() 4 problem function-size b.c:bar() 5 problem file-size header.h 8 problem include-count header.h 4 problem dependency-violation header.h 3 +practracker exit status: 11 diff --git a/scripts/maint/practracker/testdata/ex1-expected.txt b/scripts/maint/practracker/testdata/ex1-expected.txt index 58140a4d9a..2713338ae4 100644 --- a/scripts/maint/practracker/testdata/ex1-expected.txt +++ b/scripts/maint/practracker/testdata/ex1-expected.txt @@ -1,3 +1,5 @@ +Unusual pattern permitted.h in scripts/maint/practracker/testdata problem function-size a.c:i_am_a_function() 9 (warning) problem function-size a.c:another_function() 12 problem function-size b.c:foo() 4 +practracker exit status: 2 diff --git a/scripts/maint/practracker/testdata/ex1-overbroad-expected.txt b/scripts/maint/practracker/testdata/ex1-overbroad-expected.txt index f69c608f40..5ca480dc04 100644 --- a/scripts/maint/practracker/testdata/ex1-overbroad-expected.txt +++ b/scripts/maint/practracker/testdata/ex1-overbroad-expected.txt @@ -1,2 +1,4 @@ -problem file-size a.c 40 -> 38 +Unusual pattern permitted.h in scripts/maint/practracker/testdata +problem file-size a.c 45 -> 41 problem file-size z.c 100 -> 0 +practracker exit status: 3 diff --git a/scripts/maint/practracker/testdata/ex1-regen-expected.txt b/scripts/maint/practracker/testdata/ex1-regen-expected.txt new file mode 100644 index 0000000000..bdf3681edf --- /dev/null +++ b/scripts/maint/practracker/testdata/ex1-regen-expected.txt @@ -0,0 +1,46 @@ +# Welcome to the exceptions file for Tor's best-practices tracker! +# +# Each line of this file represents a single violation of Tor's best +# practices -- typically, a violation that we had before practracker.py +# first existed. +# +# There are three kinds of problems that we recognize right now: +# function-size -- a function of more than 100 lines. +# file-size -- a .c file of more than 3000 lines, or a .h +# file with more than 500 lines. +# include-count -- a .c file with more than 50 #includes, +# or a .h file with more than 15 #includes. +# dependency-violation -- a file includes a header that it should +# not, according to an advisory .may_include file. +# +# Each line below represents a single exception that practracker should +# _ignore_. Each line has four parts: +# 1. The word "problem". +# 2. The kind of problem. +# 3. The location of the problem: either a filename, or a +# filename:functionname pair. +# 4. The magnitude of the problem to ignore. +# +# So for example, consider this line: +# problem file-size /src/core/or/connection_or.c 3200 +# +# It tells practracker to allow the mentioned file to be up to 3200 lines +# long, even though ordinarily it would warn about any file with more than +# 3000 lines. +# +# You can either edit this file by hand, or regenerate it completely by +# running `make practracker-regen`. +# +# Remember: It is better to fix the problem than to add a new exception! + +problem file-size a.c 41 +problem include-count a.c 6 +problem function-size a.c:i_am_a_function() 9 +problem function-size a.c:another_function() 12 +problem dependency-violation a.c 4 +problem file-size b.c 15 +problem function-size b.c:foo() 4 +problem function-size b.c:bar() 5 +problem file-size header.h 8 +problem include-count header.h 4 +problem dependency-violation header.h 3 diff --git a/scripts/maint/practracker/testdata/ex1-regen-overbroad-expected.txt b/scripts/maint/practracker/testdata/ex1-regen-overbroad-expected.txt new file mode 100644 index 0000000000..4521029b10 --- /dev/null +++ b/scripts/maint/practracker/testdata/ex1-regen-overbroad-expected.txt @@ -0,0 +1,45 @@ +# Welcome to the exceptions file for Tor's best-practices tracker! +# +# Each line of this file represents a single violation of Tor's best +# practices -- typically, a violation that we had before practracker.py +# first existed. +# +# There are three kinds of problems that we recognize right now: +# function-size -- a function of more than 100 lines. +# file-size -- a .c file of more than 3000 lines, or a .h +# file with more than 500 lines. +# include-count -- a .c file with more than 50 #includes, +# or a .h file with more than 15 #includes. +# dependency-violation -- a file includes a header that it should +# not, according to an advisory .may_include file. +# +# Each line below represents a single exception that practracker should +# _ignore_. Each line has four parts: +# 1. The word "problem". +# 2. The kind of problem. +# 3. The location of the problem: either a filename, or a +# filename:functionname pair. +# 4. The magnitude of the problem to ignore. +# +# So for example, consider this line: +# problem file-size /src/core/or/connection_or.c 3200 +# +# It tells practracker to allow the mentioned file to be up to 3200 lines +# long, even though ordinarily it would warn about any file with more than +# 3000 lines. +# +# You can either edit this file by hand, or regenerate it completely by +# running `make practracker-regen`. +# +# Remember: It is better to fix the problem than to add a new exception! + +problem file-size a.c 41 +problem include-count a.c 6 +problem function-size a.c:i_am_a_function() 8 +problem function-size a.c:another_function() 11 +problem file-size b.c 15 +problem function-size b.c:bar() 5 +problem dependency-violation a.c 4 +problem dependency-violation header.h 3 +problem file-size header.h 8 +problem include-count header.h 4 diff --git a/scripts/maint/practracker/testdata/ex1.txt b/scripts/maint/practracker/testdata/ex1.txt index c698005d07..af8de03291 100644 --- a/scripts/maint/practracker/testdata/ex1.txt +++ b/scripts/maint/practracker/testdata/ex1.txt @@ -1,6 +1,6 @@ -problem file-size a.c 40 -problem include-count a.c 4 +problem file-size a.c 45 +problem include-count a.c 6 # this problem will produce an error problem function-size a.c:i_am_a_function() 8 # this problem will produce a warning @@ -12,7 +12,7 @@ problem file-size b.c 15 problem file-size z.c 100 problem function-size b.c:bar() 5 -problem dependency-violation a.c 3 +problem dependency-violation a.c 4 problem dependency-violation header.h 3 problem file-size header.h 8 problem include-count header.h 4 diff --git a/scripts/maint/practracker/util.py b/scripts/maint/practracker/util.py index df629110c2..c52ca2fbbf 100644 --- a/scripts/maint/practracker/util.py +++ b/scripts/maint/practracker/util.py @@ -1,3 +1,8 @@ +# Future imports for Python 2.7, mandatory in 3.0 +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + import os # We don't want to run metrics for unittests, automatically-generated C files, @@ -35,6 +40,12 @@ def get_tor_c_files(tor_topdir, include_dirs=None): continue if filename in EXCLUDE_FILES: continue + # Avoid editor temporary files + bname = os.path.basename(filename) + if bname.startswith("."): + continue + if bname.startswith("#"): + continue full_path = os.path.join(root,filename) diff --git a/scripts/maint/rectify_include_paths.py b/scripts/maint/rectify_include_paths.py index 1140e8cd22..111cf816ce 100755 --- a/scripts/maint/rectify_include_paths.py +++ b/scripts/maint/rectify_include_paths.py @@ -1,5 +1,10 @@ #!/usr/bin/python +# Future imports for Python 2.7, mandatory in 3.0 +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + import os import os.path import re diff --git a/scripts/maint/redox.py b/scripts/maint/redox.py index 203cce0107..3ad3e3f1b8 100755 --- a/scripts/maint/redox.py +++ b/scripts/maint/redox.py @@ -29,6 +29,19 @@ # "mv fname.c.newdoc fname.c". Otherwise, you'll need to merge # the parts you like by hand. +# Future imports for Python 2.7, mandatory in 3.0 +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import re +import sys + +try: + xrange # Python 2 +except NameError: + xrange = range # Python 3 + # Which files should we ignore warning from? Mostly, these are external # files that we've snarfed in from somebody else, whose C we do no intend # to document for them. @@ -52,9 +65,6 @@ ADD_DOCDOCS_TO_TYPES += [ 'variable', ] # ==================== # The rest of this should not need hacking. -import re -import sys - KINDS = [ "type", "field", "typedef", "define", "function", "variable", "enumeration" ] @@ -73,7 +83,7 @@ def parsething(thing): else: m = THING_RE.match(thing) if not m: - print thing, "???? Format didn't match." + print(thing, "???? Format didn't match.") return None, None else: name, tp, parent = m.groups() @@ -150,7 +160,7 @@ def checkf(fn, errs): """ for skip in SKIP_FILES: if fn.endswith(skip): - print "Skipping",fn + print("Skipping",fn) return comments = [] @@ -169,8 +179,8 @@ def checkf(fn, errs): ln = findline(lines, line, name) if ln == None: - print "Couldn't find the definition of %s allegedly on %s of %s"%( - name, line, fn) + print("Couldn't find the definition of %s allegedly on %s of %s"%( + name, line, fn)) else: if hasdocdoc(lines, line, kind): # print "Has a DOCDOC" @@ -215,12 +225,12 @@ def applyComments(fn, entries): outf.write(line) outf.close() - print "Added %s DOCDOCs to %s" %(N, fn) + print("Added %s DOCDOCs to %s" %(N, fn)) e = read() for fn, errs in e.iteritems(): - print `(fn, errs)` + print(repr((fn, errs))) comments = checkf(fn, errs) if comments: applyComments(fn, comments) diff --git a/scripts/maint/rename_c_identifier.py b/scripts/maint/rename_c_identifier.py new file mode 100755 index 0000000000..6e0c1d8cf1 --- /dev/null +++ b/scripts/maint/rename_c_identifier.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2001 Matej Pfajfar. +# Copyright (c) 2001-2004, Roger Dingledine. +# Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. +# Copyright (c) 2007-2019, The Tor Project, Inc. +# See LICENSE for licensing information + +""" +Helpful script to replace one or more C identifiers, and optionally +generate a commit message explaining what happened. +""" + +# Future imports for Python 2.7, mandatory in 3.0 +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +import fileinput +import os +import re +import shlex +import subprocess +import sys +import tempfile + +TOPDIR = "src" + + +def is_c_file(fn): + """ + Return true iff fn is the name of a C file. + + >>> is_c_file("a/b/module.c") + True + >>> is_c_file("a/b/module.h") + True + >>> is_c_file("a/b/module.c~") + False + >>> is_c_file("a/b/.module.c") + False + >>> is_c_file("a/b/module.cpp") + False + """ + fn = os.path.split(fn)[1] + if fn.startswith("."): + return False + ext = os.path.splitext(fn)[1] + return ext in {".c", ".h", ".i", ".inc"} + + +def list_c_files(topdir=TOPDIR): + """ + Use git to list all the C files under version control. + + >>> lst = list(list_c_files()) + >>> "src/core/mainloop/mainloop.c" in lst + True + >>> "src/core/mainloop/twiddledeedoo.c" in lst + False + >>> "micro-revision.i" in lst + False + """ + proc = subprocess.Popen( + ["git", "ls-tree", "--name-only", "-r", "HEAD", topdir], + stdout=subprocess.PIPE, + encoding="utf-8") + for line in proc.stdout.readlines(): + line = line.strip() + if is_c_file(line): + yield line + + +class Rewriter: + """ + A rewriter applies a series of word-by-word replacements, in + sequence. Replacements only happen at "word boundaries", + as determined by the \\b regular expression marker. + + ("A word is defined as a sequence of alphanumeric or underscore + characters", according to the documentation.) + + >>> R = Rewriter([("magic", "secret"), ("words", "codes")]) + >>> R.apply("The magic words are rambunctious bluejay") + 'The secret codes are rambunctious bluejay' + >>> R.apply("The magical words are rambunctious bluejay") + 'The magical codes are rambunctious bluejay' + >>> R.get_count() + 3 + + """ + + def __init__(self, replacements): + """Make a new Rewriter. Takes a sequence of pairs of + (from_id, to_id), where from_id is an identifier to replace, + and to_id is its replacement. + """ + self._patterns = [] + for id1, id2 in replacements: + pat = re.compile(r"\b{}\b".format(re.escape(id1))) + self._patterns.append((pat, id2)) + + self._count = 0 + + def apply(self, line): + """Return `line` as transformed by this rewriter.""" + for pat, ident in self._patterns: + line, count = pat.subn(ident, line) + self._count += count + return line + + def get_count(self): + """Return the number of identifiers that this rewriter has + rewritten.""" + return self._count + + +def rewrite_files(files, rewriter): + """ + Apply `rewriter` to every file in `files`, replacing those files + with their rewritten contents. + """ + for line in fileinput.input(files, inplace=True): + sys.stdout.write(rewriter.apply(line)) + + +def make_commit_msg(pairs, no_verify): + """Return a commit message to explain what was replaced by the provided + arguments. + """ + script = ["./scripts/maint/rename_c_identifier.py"] + for id1, id2 in pairs: + qid1 = shlex.quote(id1) + qid2 = shlex.quote(id2) + script.append(" {} {}".format(qid1, qid2)) + script = " \\\n".join(script) + + if len(pairs) == 1: + line1 = "Rename {} to {}".format(*pairs[0]) + else: + line1 = "Replace several C identifiers." + + msg = """\ +{} + +This is an automated commit, generated by this command: + +{} +""".format(line1, script) + + if no_verify: + msg += """ +It was generated with --no-verify, so it probably breaks some commit hooks. +The commiter should be sure to fix them up in a subsequent commit. +""" + + return msg + + +def commit(pairs, no_verify=False): + """Try to commit the current git state, generating the commit message as + appropriate. If `no_verify` is True, pass the --no-verify argument to + git commit. + """ + args = [] + if no_verify: + args.append("--no-verify") + + # We have to use a try block to delete the temporary file here, since we + # are using tempfile with delete=False. We have to use delete=False, + # since otherwise we are not guaranteed to be able to give the file to + # git for it to open. + fname = None + try: + with tempfile.NamedTemporaryFile(mode="w", delete=False) as f: + fname = f.name + f.write(make_commit_msg(pairs, no_verify)) + s = subprocess.run(["git", "commit", "-a", "-F", fname, "--edit"]+args) + if s.returncode != 0 and not no_verify: + print('"git commit" failed. Maybe retry with --no-verify?', + file=sys.stderr) + revert_changes() + return False + finally: + os.unlink(fname) + + return True + + +def any_uncommitted_changes(): + """Return True if git says there are any uncommitted changes in the current + working tree; false otherwise. + """ + s = subprocess.run(["git", "diff-index", "--quiet", "HEAD"]) + return s.returncode != 0 + + +DESC = "Replace one identifier with another throughout our source." +EXAMPLES = """\ +Examples: + + rename_c_identifier.py set_ctrl_id set_controller_id + (Replaces every occurrence of "set_ctrl_id" with "set_controller_id".) + + rename_c_identifier.py --commit set_ctrl_id set_controller_id + (As above, but also generate a git commit with an appropriate message.) + + rename_c_identifier.py a b c d + (Replace "a" with "b", and "c" with "d".)""" + + +def revert_changes(): + """Tell git to revert all the changes in the current working tree. + """ + print('Reverting changes.', file=sys.stderr) + subprocess.run(["git", "checkout", "--quiet", TOPDIR]) + + +def main(argv): + import argparse + parser = argparse.ArgumentParser(description=DESC, epilog=EXAMPLES, + # prevent re-wrapping the examples + formatter_class=argparse.RawDescriptionHelpFormatter) + + parser.add_argument("--commit", action='store_true', + help="Generate a Git commit.") + parser.add_argument("--no-verify", action='store_true', + help="Tell Git not to run its pre-commit hooks.") + parser.add_argument("from_id", type=str, help="Original identifier") + parser.add_argument("to_id", type=str, help="New identifier") + parser.add_argument("more", type=str, nargs=argparse.REMAINDER, + help="Additional identifier pairs") + + args = parser.parse_args(argv[1:]) + + if len(args.more) % 2 != 0: + print("I require an even number of identifiers.", file=sys.stderr) + return 1 + + if any_uncommitted_changes(): + print("Uncommitted changes found. Not running.", file=sys.stderr) + return 1 + + pairs = [] + print("renaming {} to {}".format(args.from_id, args.to_id), file=sys.stderr) + pairs.append((args.from_id, args.to_id)) + for idx in range(0, len(args.more), 2): + id1 = args.more[idx] + id2 = args.more[idx+1] + print("renaming {} to {}".format(id1, id2)) + pairs.append((id1, id2)) + + rewriter = Rewriter(pairs) + + rewrite_files(list_c_files(), rewriter) + + print("Replaced {} identifiers".format(rewriter.get_count()), + file=sys.stderr) + + if args.commit: + commit(pairs, args.no_verify) + + +if __name__ == '__main__': + main(sys.argv) diff --git a/scripts/maint/sortChanges.py b/scripts/maint/sortChanges.py index 986b94b025..2e049b1e53 100755 --- a/scripts/maint/sortChanges.py +++ b/scripts/maint/sortChanges.py @@ -7,6 +7,11 @@ changelog. """ +# Future imports for Python 2.7, mandatory in 3.0 +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + import re import sys @@ -43,7 +48,7 @@ REPLACEMENTS = { def score(s,fname=None): m = re.match(r'^ +o ([^\n]*)\n(.*)', s, re.M|re.S) if not m: - print >>sys.stderr, "Can't score %r from %s"%(s,fname) + print("Can't score %r from %s"%(s,fname), file=sys.stderr) heading = m.group(1) heading = REPLACEMENTS.get(heading, heading) lw = m.group(1).lower() @@ -100,9 +105,9 @@ changes.sort() last_lw = "this is not a header" for _, lw, header, rest in changes: if lw == last_lw: - print rest, + print(rest, end="") else: - print - print " o",header - print rest, + print() + print(" o",header) + print(rest, end="") last_lw = lw diff --git a/scripts/maint/update_versions.py b/scripts/maint/update_versions.py index 8067f2c6c8..07de1c343a 100755 --- a/scripts/maint/update_versions.py +++ b/scripts/maint/update_versions.py @@ -1,6 +1,9 @@ #!/usr/bin/env python +# Future imports for Python 2.7, mandatory in 3.0 +from __future__ import division from __future__ import print_function +from __future__ import unicode_literals import io import os @@ -95,7 +98,7 @@ def update_file(fname, replace_on_change(fname, have_changed) # Find out our version -with open("configure.ac") as f: +with open(P("configure.ac")) as f: version = find_version(f) # If we have no version, we can't proceed. diff --git a/scripts/test/appveyor-irc-notify.py b/scripts/test/appveyor-irc-notify.py index cfe0afe7ae..598a68f47d 100644 --- a/scripts/test/appveyor-irc-notify.py +++ b/scripts/test/appveyor-irc-notify.py @@ -75,8 +75,10 @@ in Appveyor's YAML: - "python scripts/test/appveyor-irc-notify.py irc.oftc.net:6697 tor-ci failure """ +# Future imports for Python 2.7, mandatory in 3.0 +from __future__ import division from __future__ import print_function -from __future__ import absolute_import +from __future__ import unicode_literals import os import random |