summaryrefslogtreecommitdiff
path: root/scripts
diff options
context:
space:
mode:
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/build/combine_libs35
-rwxr-xr-xscripts/ci/ci-driver.sh2
-rwxr-xr-xscripts/codegen/fuzzing_include_am.py9
-rw-r--r--scripts/codegen/makedesc.py115
-rwxr-xr-xscripts/git/git-install-tools.sh6
-rwxr-xr-xscripts/git/git-list-tor-branches.sh9
-rwxr-xr-xscripts/git/git-pull-all.sh16
-rwxr-xr-xscripts/git/git-resquash.sh46
-rwxr-xr-xscripts/git/git-setup-dirs.sh31
-rwxr-xr-xscripts/git/pre-commit.git-hook26
-rwxr-xr-xscripts/git/pre-push.git-hook1
-rwxr-xr-xscripts/maint/annotate_ifdef_directives.py2
-rw-r--r--scripts/maint/checkOptionDocs.pl.in2
-rwxr-xr-xscripts/maint/clang-format.sh41
-rwxr-xr-xscripts/maint/code-format.sh232
-rwxr-xr-xscripts/maint/codetool.py2
-rw-r--r--scripts/maint/practracker/.enable_practracker_in_hooks1
-rw-r--r--scripts/maint/practracker/exceptions.txt37
-rwxr-xr-xscripts/maint/rename_c_identifier.py4
19 files changed, 489 insertions, 128 deletions
diff --git a/scripts/build/combine_libs b/scripts/build/combine_libs
new file mode 100755
index 0000000000..fb311552fe
--- /dev/null
+++ b/scripts/build/combine_libs
@@ -0,0 +1,35 @@
+#!/bin/sh
+
+set -e
+
+TMPDIR="$(mktemp -d -t tor_lib_combining.XXXXXX)"
+ORIGDIR="$(pwd)"
+
+trap 'cd "$ORIGDIR" && rm -rf "$TMPDIR"' 0
+
+abspath() {
+ echo "$(cd "$(dirname "$1")">/dev/null && pwd)/$(basename "$1")"
+}
+
+TARGET=$(abspath "$1")
+
+#echo ORIGDIR="$ORIGDIR"
+#echo AR="$AR"
+#echo ARFLAGS="$AFLAGS"
+
+shift
+
+for input in "$@"; do
+ cd "$ORIGDIR"
+ abs=$(abspath "$input")
+ dir="$TMPDIR"/$(basename "$input" .a)
+ mkdir "$dir"
+ cd "$dir">/dev/null
+ ar x "$abs"
+done
+
+cd "$TMPDIR" >/dev/null
+#echo "${AR:-ar}" "${ARFLAGS:-cru}" library.tmp.a ./*/**
+"${AR:-ar}" "${ARFLAGS:-cru}" library.tmp.a ./*/**
+"${RANLIB:-ranlib}" library.tmp.a
+mv -f library.tmp.a "$TARGET"
diff --git a/scripts/ci/ci-driver.sh b/scripts/ci/ci-driver.sh
index 67a95a1f61..c07d99188b 100755
--- a/scripts/ci/ci-driver.sh
+++ b/scripts/ci/ci-driver.sh
@@ -461,7 +461,7 @@ fi
if [[ "${STEM}" = "yes" ]]; then
start_section "Stem"
if [[ "${TOR_VER_AT_LEAST_044}" = 'yes' ]]; then
- # XXXX This shold probably be part some test-stem make target.
+ # XXXX This should probably be part of some test-stem make target.
if runcmd timelimit -p -t 520 -s USR1 -T 30 -S ABRT \
python3 "${STEM_PATH}/run_tests.py" \
--tor src/app/tor \
diff --git a/scripts/codegen/fuzzing_include_am.py b/scripts/codegen/fuzzing_include_am.py
index ae50563074..b52b956f81 100755
--- a/scripts/codegen/fuzzing_include_am.py
+++ b/scripts/codegen/fuzzing_include_am.py
@@ -31,17 +31,18 @@ FUZZING_CFLAGS = \
FUZZING_LDFLAG = \
@TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) @TOR_LDFLAGS_libevent@
FUZZING_LIBS = \
- $(TOR_INTERNAL_TESTING_LIBS) \
+ src/test/libtor-testing.a \
$(rust_ldadd) \
@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \
@TOR_LIBEVENT_LIBS@ $(TOR_LIBS_CRYPTLIB) \
- @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ @CURVE25519_LIBS@ \
+ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ @CURVE25519_LIBS@ \
@TOR_SYSTEMD_LIBS@ \
@TOR_LZMA_LIBS@ \
- @TOR_ZSTD_LIBS@
+ @TOR_ZSTD_LIBS@ \
+ @TOR_TRACE_LIBS@
oss-fuzz-prereqs: \
- $(TOR_INTERNAL_TESTING_LIBS)
+ src/test/libtor-testing.a
noinst_HEADERS += \
src/test/fuzz/fuzzing.h
diff --git a/scripts/codegen/makedesc.py b/scripts/codegen/makedesc.py
index 48d1d31a02..5c59a52af1 100644
--- a/scripts/codegen/makedesc.py
+++ b/scripts/codegen/makedesc.py
@@ -70,19 +70,39 @@ i2d_RSAPublicKey.argtypes = [
i2d_RSAPublicKey.restype = ctypes.c_int
+HEADER = """\
+router fred 127.0.0.1 9001 0 9002
+identity-ed25519
+{d.ED_CERT}
+signing-key
+{d.RSA_IDENTITY}
+master-key-ed25519 {d.ED_IDENTITY}
+onion-key
+{d.RSA_ONION_KEY}
+ntor-onion-key {d.NTOR_ONION_KEY}
+ntor-onion-key-crosscert {d.NTOR_CROSSCERT_SIGN}
+{d.NTOR_CROSSCERT}
+onion-key-crosscert
+{d.RSA_CROSSCERT_ED}
+"""
+
+FOOTER="""
+
+"""
+
def rsa_sign(msg, rsa):
- buf = ctypes.create_string_buffer(1024)
+ buf = ctypes.create_string_buffer(2048)
n = RSA_private_encrypt(len(msg), msg, buf, rsa, 1)
if n <= 0:
raise Exception()
return buf.raw[:n]
-def b64(x):
- x = base64.b64encode(x)
+def b64(x1):
+ x = binascii.b2a_base64(x1)
res = []
for i in xrange(0, len(x), 64):
- res.append(x[i:i+64]+"\n")
- return "".join(res)
+ res.append((x[i:i+64]).decode("ascii"))
+ return "\n".join(res)
def bio_extract(bio):
buf = ctypes.c_char_p()
@@ -100,18 +120,19 @@ def make_rsa_key(e=65537):
n = crypt.i2d_RSAPublicKey(rsa, ctypes.byref(pBuf))
s = buf.raw[:n]
digest = hashlib.sha1(s).digest()
+ pem = pem.decode("ascii")
return (rsa,pem,digest)
def makeEdSigningKeyCert(sk_master, pk_master, pk_signing, date,
includeSigning=False, certType=1):
assert len(pk_signing) == len(pk_master) == 32
- expiration = struct.pack("!L", date//3600)
+ expiration = struct.pack(b"!L", date//3600)
if includeSigning:
- extensions = "\x01\x00\x20\x04\x00%s"%(pk_master)
+ extensions = b"\x01\x00\x20\x04\x00%s"%(pk_master)
else:
- extensions = "\x00"
- signed = "\x01%s%s\x01%s%s" % (
- chr(certType), expiration, pk_signing, extensions)
+ extensions = b"\x00"
+ signed = b"\x01%s%s\x01%s%s" % (
+ bytes([certType]), expiration, pk_signing, extensions)
signature = ed25519_exts_ref.signatureWithESK(signed, sk_master, pk_master)
assert len(signature) == 64
return signed+signature
@@ -127,7 +148,7 @@ MAGIC2 = "<<<<<!#!#!#XYZZY#!#!#!>>>>>"
class OnDemandKeys(object):
def __init__(self, certDate=None):
if certDate is None:
- certDate = time.time() + 86400
+ certDate = int(time.time()) + 86400
self.certDate = certDate
self.rsa_id = None
self.rsa_onion_key = None
@@ -151,7 +172,7 @@ class OnDemandKeys(object):
@property
def RSA_FINGERPRINT_NOSPACE(self):
- return binascii.b2a_hex(self.RSA_ID_DIGEST).upper()
+ return binascii.b2a_hex(self.RSA_ID_DIGEST).upper().decode("ascii")
@property
def RSA_ONION_KEY(self):
@@ -162,7 +183,7 @@ class OnDemandKeys(object):
@property
def RSA_FINGERPRINT(self):
- hexdigest = self.RSA_FINGERPRINT_NOSPACEK
+ hexdigest = self.RSA_FINGERPRINT_NOSPACE
return " ".join(hexdigest[i:i+4] for i in range(0,len(hexdigest),4))
@property
@@ -178,7 +199,7 @@ class OnDemandKeys(object):
if self.ntor_sk is None:
self.ntor_sk = slownacl_curve25519.Private()
self.ntor_pk = self.ntor_sk.get_public()
- return base64.b64encode(self.ntor_pk.serialize())
+ return base64.b64encode(self.ntor_pk.serialize()).decode("ascii")
@property
def ED_CERT(self):
@@ -192,6 +213,11 @@ class OnDemandKeys(object):
return objwrap('ED25519 CERT', b64(self.ed_cert))
@property
+ def ED_IDENTITY(self):
+ self.ED_CERT
+ return binascii.b2a_base64(self.ed_id_pk).strip().decode("ascii")
+
+ @property
def NTOR_CROSSCERT(self):
if self.ntor_crosscert is None:
self.ED_CERT
@@ -199,7 +225,7 @@ class OnDemandKeys(object):
ed_privkey = self.ntor_sk.serialize() + os.urandom(32)
ed_pub0 = ed25519_exts_ref.publickeyFromESK(ed_privkey)
- sign = (ord(ed_pub0[31]) & 255) >> 7
+ sign = ((ed_pub0[31]) & 255) >> 7
self.ntor_crosscert = makeEdSigningKeyCert(self.ntor_sk.serialize() + os.urandom(32), ed_pub0, self.ed_id_pk, self.certDate, certType=10)
self.ntor_crosscert_sign = sign
@@ -234,18 +260,19 @@ class OnDemandKeys(object):
self.ED_CERT
signed_part = body[:idx+len("\nrouter-sig-ed25519 ")]
signed_part = "Tor router descriptor signature v1" + signed_part
- digest = hashlib.sha256(signed_part).digest()
+ digest = hashlib.sha256(signed_part.encode("utf-8")).digest()
ed_sig = ed25519_exts_ref.signatureWithESK(digest,
self.ed_signing_sk, self.ed_signing_pk)
- body = body.replace(MAGIC2, base64.b64encode(ed_sig).replace("=",""))
+ body = body.replace(MAGIC2, base64.b64encode(ed_sig).decode("ascii").replace("=",""))
+ self.RSA_IDENTITY
idx = body.rindex("\nrouter-signature")
end_of_sig = body.index("\n", idx+1)
signed_part = body[:end_of_sig+1]
- digest = hashlib.sha1(signed_part).digest()
+ digest = hashlib.sha1(signed_part.encode("utf-8")).digest()
assert len(digest) == 20
rsasig = rsa_sign(digest, self.rsa_id)
@@ -318,29 +345,42 @@ def emit_ri(name, body):
body = info.sign_desc(body)
print_c_string("EX_RI_%s"%name.upper(), body)
-def emit_ei(name, body):
+def emit_ei(name, body, fields):
info = OnDemandKeys()
body = body.format(d=info)
body = info.sign_desc(body)
print_c_string("EX_EI_%s"%name.upper(), body)
- print('const char EX_EI_{NAME}_FP[] = "{d.RSA_FINGERPRINT_NOSPACE}";'.format(
+ print('ATTR_UNUSED static const char EX_EI_{NAME}_FP[] = "{d.RSA_FINGERPRINT_NOSPACE}";'.format(
d=info, NAME=name.upper()))
+ print("ATTR_UNUSED")
print_c_string("EX_EI_%s_KEY"%name.upper(), info.RSA_IDENTITY)
def analyze(s):
- fields = {}
- while s.startswith(":::"):
- first,s=s.split("\n", 1)
- m = re.match(r'^:::(\w+)=(.*)',first)
- if not m:
- raise ValueError(first)
- k,v = m.groups()
- fields[k] = v
- return fields, s
-
-def process_file(s):
- fields, s = analyze(s)
+ while s:
+ fields = {}
+ s_pre = s
+ while s.startswith(":::"):
+ first,s=s.split("\n", 1)
+ m = re.match(r'^:::(\w+)=(.*)',first)
+ if not m:
+ raise ValueError(first)
+ k,v = m.groups()
+ fields[k] = v
+ if "name" not in fields:
+ print(repr(s_pre))
+
+ idx = s.find(":::")
+ if idx != -1:
+ body = s[:idx].rstrip()
+ s = s[idx:]
+ else:
+ body = s.rstrip()
+ s = ""
+
+ yield (fields, body)
+
+def emit_entry(fields, s):
try:
name = fields['name']
tp = fields['type']
@@ -348,12 +388,21 @@ def process_file(s):
raise ValueError("missing required field")
if tp == 'ei':
- emit_ei(name, s)
+ emit_ei(name, s, fields)
elif tp == 'ri':
emit_ri(name, s)
else:
raise ValueError("unrecognized type")
+def process_file(s):
+ print("""\
+/* These entries are automatically generated by makedesc.py to make sure
+ * that their keys and signatures are right except when otherwise
+ * specified. */
+""")
+ for (fields, s) in analyze(s):
+ emit_entry(fields, s)
+
if __name__ == '__main__':
import sys
for fn in sys.argv[1:]:
diff --git a/scripts/git/git-install-tools.sh b/scripts/git/git-install-tools.sh
index ef8623a018..d74f8475af 100755
--- a/scripts/git/git-install-tools.sh
+++ b/scripts/git/git-install-tools.sh
@@ -3,7 +3,7 @@
SCRIPT_NAME=$(basename "$0")
SCRIPTS_DIR=$(dirname "$0")
-TOOL_NAMES=(push-all pull-all merge-forward list-tor-branches)
+TOOL_NAMES=(push-all pull-all merge-forward list-tor-branches resquash)
function usage()
{
@@ -146,7 +146,7 @@ if [[ $INSTALL_HOOKS = 1 ]]; then
note "Installing hooks"
for fn in "$SCRIPTS_DIR"/*.git-hook; do
name=$(basename "$fn")
- $RUN install --backup "$fn" "${HOOKS_DIR}/${name%.git-hook}"
+ $RUN install -b "$fn" "${HOOKS_DIR}/${name%.git-hook}"
done
fi
@@ -163,7 +163,7 @@ if [[ $INSTALL_TOOLS = 1 ]]; then
note "Copying scripts"
for tool in "${TOOL_NAMES[@]}"; do
- $RUN install --backup "${SCRIPTS_DIR}/git-${tool}.sh" "${TOR_DEVTOOLS_DIR}/"
+ $RUN install -b "${SCRIPTS_DIR}/git-${tool}.sh" "${TOR_DEVTOOLS_DIR}/"
done
fi
diff --git a/scripts/git/git-list-tor-branches.sh b/scripts/git/git-list-tor-branches.sh
index d6b30f064f..5a527ffc05 100755
--- a/scripts/git/git-list-tor-branches.sh
+++ b/scripts/git/git-list-tor-branches.sh
@@ -139,15 +139,12 @@ finish() {
branch maint-0.3.5
branch release-0.3.5
-branch maint-0.4.1
-branch release-0.4.1
-
-branch maint-0.4.2
-branch release-0.4.2
-
branch maint-0.4.3
branch release-0.4.3
+branch maint-0.4.4
+branch release-0.4.4
+
branch master
finish
diff --git a/scripts/git/git-pull-all.sh b/scripts/git/git-pull-all.sh
index 7f82eda296..52a5c6140c 100755
--- a/scripts/git/git-pull-all.sh
+++ b/scripts/git/git-pull-all.sh
@@ -181,6 +181,19 @@ function fetch_tor_github
fi
}
+# Fetch tor-gitlab pull requests. No arguments.
+function fetch_tor_gitlab
+{
+ local cmd="git fetch tor-gitlab"
+ printf " %s Fetching tor-gitlab..." "$MARKER"
+ if [ $DRY_RUN -eq 0 ]; then
+ msg=$( eval "$cmd" 2>&1 )
+ validate_ret $? "$msg"
+ else
+ printf "\\n %s\\n" "${IWTH}$cmd${CNRM}"
+ fi
+}
+
###############
# Entry point #
###############
@@ -189,6 +202,9 @@ function fetch_tor_github
goto_repo "$ORIGIN_PATH"
fetch_tor_github
+# Then tor-gitlab
+fetch_tor_gitlab
+
# Then, fetch the origin.
fetch_origin
diff --git a/scripts/git/git-resquash.sh b/scripts/git/git-resquash.sh
new file mode 100755
index 0000000000..e0f26ecdc4
--- /dev/null
+++ b/scripts/git/git-resquash.sh
@@ -0,0 +1,46 @@
+#!/bin/sh
+#
+# Provides a convenient alias for "git rebase -i --autosquash --keep-root"
+# on gits that have it, and a replacement on gits that don't.
+
+set -e
+
+PARENT="$1"
+
+if test "x$PARENT" = "x"; then
+ echo "You must specify the parent branch."
+ exit 1
+fi
+
+# Can we use git rebase --keep-base? Detect the git version to find out.
+GITVER=$(git version)
+if test "$(echo "$GITVER"|cut -d ' ' -f 1-2)" = "git version"; then
+ # --keep-base was added in git 2.24. Detect if we have that version.
+ GITVER=$(echo "$GITVER" | cut -d ' ' -f 3)
+ major=$(echo "$GITVER" | cut -d . -f 1)
+ minor=$(echo "$GITVER" | cut -d . -f 2)
+ if test "$major" -lt 2; then
+ USE_KEEP_BASE=0
+ elif test "$major" -eq 2 && test "$minor" -lt 24; then
+ USE_KEEP_BASE=0
+ else
+ USE_KEEP_BASE=1
+ fi
+else
+ # This isn't a git that reports its version in a way recognize; assume that
+ # --keep-base will work
+ USE_KEEP_BASE=1
+fi
+
+if test "x$USE_KEEP_BASE" = "x1" ; then
+ exec git rebase -i --autosquash --keep-base "${PARENT}"
+else
+ REV=$(git log --reverse --format='%H' "${PARENT}..HEAD" | head -1)
+
+ if test "x${REV}" = "x"; then
+ echo "No changes here since ${PARENT}"
+ exit 1
+ fi
+
+ exec git rebase -i --autosquash "${REV}^"
+fi
diff --git a/scripts/git/git-setup-dirs.sh b/scripts/git/git-setup-dirs.sh
index 1f61eb8b83..5a9ae41cbd 100755
--- a/scripts/git/git-setup-dirs.sh
+++ b/scripts/git/git-setup-dirs.sh
@@ -40,6 +40,10 @@ function usage()
echo " (current: $GITHUB_PULL)"
echo " TOR_GITHUB_PUSH: the tor-github remote push URL"
echo " (current: $GITHUB_PUSH)"
+ echo " TOR_GITLAB_PULL: the tor-gitlab remote pull URL"
+ echo " (current: $GITLAB_PULL)"
+ echo " TOR_GITLAB_PUSH: the tor-gitlab remote push URL"
+ echo " (current: $GITLAB_PUSH)"
echo " TOR_EXTRA_CLONE_ARGS: extra arguments to git clone"
echo " (current: $TOR_EXTRA_CLONE_ARGS)"
echo " TOR_EXTRA_REMOTE_NAME: the name of an extra remote"
@@ -83,6 +87,10 @@ fi
GITHUB_PULL=${TOR_GITHUB_PULL:-"https://github.com/torproject/tor.git"}
GITHUB_PUSH=${TOR_GITHUB_PUSH:-"No_Pushing_To_GitHub"}
+# GitLab repositories
+GITLAB_PULL=${TOR_GITLAB_PULL:-"https://gitlab.torproject.org/tpo/core/tor.git"}
+GITLAB_PUSH=${TOR_GITLAB_PUSH:-"No_Pushing_To_GitLab"}
+
##########################
# Git branches to manage #
##########################
@@ -343,6 +351,20 @@ function set_tor_github_pr_fetch_config
"refs/pull.*pr"
}
+# Set up the tor-github PR config, so tor-gitlab/mr/NNNN points to GitHub
+# MR NNNN. In some repositories, "/head" is optional.
+function set_tor_gitlab_mr_fetch_config
+{
+ # standard branches
+ replace_fetch_config tor-gitlab \
+ "+refs/heads/*:refs/remotes/tor-gitlab/*" \
+ "refs/heads"
+ # MRs
+ replace_fetch_config tor-gitlab \
+ "+refs/merge-requests/*/head:refs/remotes/tor-gitlab/mr/*" \
+ "refs/merge-requests.*mr"
+}
+
# Add a new worktree for branch at path.
# If the directory already exists: fail if $USE_EXISTING is 0, otherwise skip.
function add_worktree
@@ -471,6 +493,15 @@ set_tor_github_pr_fetch_config
# Now fetch them all
fetch_remote "tor-github"
+# GitLab remote
+printf "%s Setting up remote %s\\n" "$MARKER" "${BYEL}tor-gitlab${CNRM}"
+add_remote "tor-gitlab" "$GITLAB_PULL"
+set_remote_push "tor-gitlab" "$GITLAB_PUSH"
+# Add custom fetch for MRs
+set_tor_gitlab_mr_fetch_config
+# Now fetch them all
+fetch_remote "tor-gitlab"
+
# Extra remote
if [ "$TOR_EXTRA_REMOTE_NAME" ]; then
printf "%s Setting up remote %s\\n" "$MARKER" \
diff --git a/scripts/git/pre-commit.git-hook b/scripts/git/pre-commit.git-hook
index f630a242bd..75e5133a73 100755
--- a/scripts/git/pre-commit.git-hook
+++ b/scripts/git/pre-commit.git-hook
@@ -3,9 +3,14 @@
# To install this script, copy it to .git/hooks/pre-commit in local copy of
# tor git repo and make sure it has permission to execute.
#
-# This is pre-commit git hook script that prevents commiting your changeset if
+# This is pre-commit git hook script that prevents committing your changeset if
# it fails our code formatting, changelog entry formatting, module include
-# rules, or best practices tracker.
+# rules, etc...
+
+# Run only if this environment variable is set.
+if [ -z "$TOR_EXTRA_PRE_COMMIT_CHECKS" ]; then
+ exit 0
+fi
workdir=$(git rev-parse --show-toplevel)
@@ -49,13 +54,6 @@ if [ -e scripts/maint/checkShellScripts.sh ]; then
scripts/maint/checkShellScripts.sh
fi
-# Always run the practracker unit tests
-PT_DIR=scripts/maint/practracker
-
-if [ -e "${PT_DIR}/test_practracker.sh" ]; then
- "${PT_DIR}/test_practracker.sh"
-fi
-
if [ -e scripts/maint/checkSpaceTest.sh ]; then
scripts/maint/checkSpaceTest.sh
fi
@@ -74,19 +72,11 @@ printf "Modified tor-owned source files:\\n%s\\n" "$CHECK_FILES"
perl scripts/maint/checkSpace.pl -C \
$CHECK_FILES
+# This makes sure that we are only including things we're allowed to include.
if test -e scripts/maint/practracker/includes.py; then
python scripts/maint/practracker/includes.py
fi
-# Only call practracker if ${PT_DIR}/.enable_practracker_in_hooks exists
-# We do this check so that we can enable practracker in hooks in master, and
-# disable it on maint branches
-if [ -e "${PT_DIR}/practracker.py" ]; then
- if [ -e "${PT_DIR}/.enable_practracker_in_hooks" ]; then
- python3 "${PT_DIR}/practracker.py" "$workdir"
- fi
-fi
-
if [ -e scripts/coccinelle/check_cocci_parse.sh ]; then
# Run a verbose cocci parse check on the changed files
diff --git a/scripts/git/pre-push.git-hook b/scripts/git/pre-push.git-hook
index efa45b9860..f0a3a250ec 100755
--- a/scripts/git/pre-push.git-hook
+++ b/scripts/git/pre-push.git-hook
@@ -83,6 +83,7 @@ do
src/tools/*.[ch] \
)"
+ export TOR_EXTRA_PRE_COMMIT_CHECKS=1
# We want word splitting here, because file names are space
# separated
# shellcheck disable=SC2086
diff --git a/scripts/maint/annotate_ifdef_directives.py b/scripts/maint/annotate_ifdef_directives.py
index 9ca090d595..eeca0bbb17 100755
--- a/scripts/maint/annotate_ifdef_directives.py
+++ b/scripts/maint/annotate_ifdef_directives.py
@@ -70,7 +70,7 @@ LINE_OBVIOUSNESS_LIMIT = 4
# Maximum line width. This includes a terminating newline character.
#
# (This is the maximum before encoding, so that if the the operating system
-# uses multiple characers to encode newline, that's still okay.)
+# uses multiple characters to encode newline, that's still okay.)
LINE_WIDTH=80
class Problem(Exception):
diff --git a/scripts/maint/checkOptionDocs.pl.in b/scripts/maint/checkOptionDocs.pl.in
index 6533c762c5..bb8008c2e8 100644
--- a/scripts/maint/checkOptionDocs.pl.in
+++ b/scripts/maint/checkOptionDocs.pl.in
@@ -39,7 +39,7 @@ loadTorrc("@abs_top_srcdir@/src/config/torrc.sample.in", \%torrcSampleOptions);
# Try to figure out what's in the man page.
my $considerNextLine = 0;
-open(F, "@abs_top_srcdir@/doc/tor.1.txt") or die;
+open(F, "@abs_top_srcdir@/doc/man/tor.1.txt") or die;
while (<F>) {
if (m!^(?:\[\[([A-za-z0-9_]+)\]\] *)?\*\*([A-Za-z0-9_]+)\*\*!) {
$manPageOptions{$2} = 1;
diff --git a/scripts/maint/clang-format.sh b/scripts/maint/clang-format.sh
deleted file mode 100755
index 59832117b4..0000000000
--- a/scripts/maint/clang-format.sh
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/bin/sh
-# Copyright 2020, The Tor Project, Inc.
-# See LICENSE for licensing information.
-
-#
-# DO NOT COMMIT OR MERGE CODE THAT IS RUN THROUGH THIS TOOL YET.
-#
-# WE ARE STILL DISCUSSING OUR DESIRED STYLE AND ITERATING ON IT.
-# (12 Feb 2020)
-#
-
-# This script runs "clang-format" and "codetool" in sequence over each of
-# our source files, and replaces the original file only if it has changed.
-#
-# We can't just use clang-format -i, since we also want to use codetool to
-# reformat a few things back to how we want them, and we want avoid changing
-# the mtime on files that didn't actually change.
-
-set -e
-
-cd "$(dirname "$0")/../../src/"
-
-# Shellcheck complains that a for loop over find's output is unreliable,
-# since there might be special characters in the output. But we happen
-# to know that none of our C files have special characters or spaces in
-# their names, so this is safe.
-#
-# shellcheck disable=SC2044
-for fname in $(find lib core feature app test tools -name '[^.]*.[ch]'); do
- tmpfname="${fname}.clang_fmt.tmp"
- rm -f "${tmpfname}"
- clang-format --style=file "${fname}" > "${tmpfname}"
- ../scripts/maint/codetool.py "${tmpfname}"
- if cmp "${fname}" "${tmpfname}" >/dev/null 2>&1; then
- echo "No change in ${fname}"
- rm -f "${tmpfname}"
- else
- echo "Change in ${fname}"
- mv "${tmpfname}" "${fname}"
- fi
-done
diff --git a/scripts/maint/code-format.sh b/scripts/maint/code-format.sh
new file mode 100755
index 0000000000..d8f597d70d
--- /dev/null
+++ b/scripts/maint/code-format.sh
@@ -0,0 +1,232 @@
+#!/usr/bin/env bash
+# Copyright 2020, The Tor Project, Inc.
+# See LICENSE for licensing information.
+
+#
+# DO NOT COMMIT OR MERGE CODE THAT IS RUN THROUGH THIS TOOL YET.
+#
+# WE ARE STILL DISCUSSING OUR DESIRED STYLE AND ITERATING ON IT.
+# (12 Feb 2020)
+#
+
+# This script runs "clang-format" and "codetool" in sequence over each of its
+# arguments. It either replaces the original, or says whether anything has
+# changed, depending on its arguments.
+#
+# We can't just use clang-format directly, since we also want to use codetool
+# to reformat a few things back to how we want them, and we want avoid changing
+# the mtime on files that didn't actually change.
+#
+# Use "-i" to edit the file in-place.
+# Use "-c" to exit with a nonzero exit status if any file needs to change.
+# Use "-d" to emit diffs.
+#
+# The "-a" option tells us to run over every Tor source file.
+# The "-v" option tells us to be verbose.
+
+set -e
+
+ALL=0
+GITDIFF=0
+GITIDX=0
+DIFFMODE=0
+CHECKMODE=0
+CHANGEMODE=0
+
+SCRIPT_NAME=$(basename "$0")
+SCRIPT_DIR=$(dirname "$0")
+SRC_DIR="${SCRIPT_DIR}/../../src"
+
+function usage() {
+ echo "$SCRIPT_NAME [-h] [-c|-d|-i] [-v] [-a|-G|files...]"
+ echo
+ echo " flags:"
+ echo " -h: show this help text"
+ echo " -c: check whether files are correctly formatted"
+ echo " -d: print a diff for the changes that would be applied"
+ echo " -i: change files in-place"
+ echo " -a: run over all the C files in Tor"
+ echo " -v: verbose mode"
+ echo " -g: look at the files that have changed in git."
+ echo " -G: look at the files that are staged for the git commit."
+ echo
+ echo "EXAMPLES"
+ echo
+ echo " $SCRIPT_NAME -a -i"
+ echo " rewrite every file in place, whether it has changed or not."
+ echo " $SCRIPT_NAME -a -d"
+ echo " as above, but only display the changes."
+ echo " $SCRIPT_NAME -g -i"
+ echo " update every file that you have changed in the git working tree."
+ echo " $SCRIPT_NAME -G -c"
+ echo " exit with an error if any staged changes are not well-formatted."
+}
+
+FILEARGS_OK=1
+
+while getopts "acdgGhiv" opt; do
+ case "$opt" in
+ h) usage
+ exit 0
+ ;;
+ a) ALL=1
+ FILEARGS_OK=0
+ ;;
+ g) GITDIFF=1
+ FILEARGS_OK=0
+ ;;
+ G) GITIDX=1
+ FILEARGS_OK=0
+ ;;
+ c) CHECKMODE=1
+ ;;
+ d) DIFFMODE=1
+ ;;
+ i) CHANGEMODE=1
+ ;;
+ v) VERBOSE=1
+ ;;
+ *) echo
+ usage
+ exit 1
+ ;;
+ esac
+done
+# get rid of the flags; keep the filenames.
+shift $((OPTIND - 1))
+
+# Define a verbose function.
+if [[ $VERBOSE = 1 ]]; then
+ function note()
+ {
+ echo "$@"
+ }
+else
+ function note()
+ {
+ true
+ }
+fi
+
+# We have to be in at least one mode, or we can't do anything
+if [[ $CHECKMODE = 0 && $DIFFMODE = 0 && $CHANGEMODE = 0 ]]; then
+ echo "Nothing to do. You need to specify -c, -d, or -i."
+ echo "Try $SCRIPT_NAME -h for more information."
+ exit 0
+fi
+
+# We don't want to "give an error if anything would change" if we're
+# actually trying to change things.
+if [[ $CHECKMODE = 1 && $CHANGEMODE = 1 ]]; then
+ echo "It doesn't make sense to use -c and -i together."
+ exit 0
+fi
+# It doesn't make sense to look at "all files" and "git files"
+if [[ $((ALL + GITIDX + GITDIFF)) -gt 1 ]]; then
+ echo "It doesn't make sense to use more than one of -a, -g, or -G together."
+ exit 0
+fi
+
+if [[ $FILEARGS_OK = 1 ]]; then
+ # The filenames are on the command-line.
+ INPUTS=("${@}")
+else
+ if [[ "$#" != 0 ]]; then
+ echo "Can't use -a, -g, or -G with additional command-line arguments."
+ exit 1
+ fi
+fi
+
+if [[ $ALL = 1 ]]; then
+ # We're in "all" mode -- use find(1) to find the filenames.
+ mapfile -d '' INPUTS < <(find "${SRC_DIR}"/{lib,core,feature,app,test,tools} -name '[^.]*.[ch]' -print0)
+elif [[ $GITIDX = 1 ]]; then
+ # We're in "git index" mode -- use git diff --cached to find the filenames
+ # that are changing in the index, then strip out the ones that
+ # aren't C.
+ mapfile INPUTS < <(git diff --name-only --cached --diff-filter=AMCR | grep '\.[ch]$')
+elif [[ $GITDIFF = 1 ]]; then
+ # We are in 'git diff' mode -- we want everything that changed, including
+ # the index and the working tree.
+ #
+ # TODO: There might be a better way to do this.
+ mapfile INPUTS < <(git diff --name-only --cached --diff-filter=AMCR | grep '\.[ch]$'; git diff --name-only --diff-filter=AMCR | grep '\.[ch]$' )
+fi
+
+if [[ $GITIDX = 1 ]]; then
+ # If we're running in git mode, we need to stash all the changes that
+ # we don't want to look at. This is necessary even though we're only
+ # looking at the changed files, since we might have the file only
+ # partially staged.
+ note "Stashing unstaged changes"
+ git stash -q --keep-index
+ function restoregit() {
+ note "Restoring git state"
+ git stash pop -q
+ }
+else
+ function restoregit() {
+ true
+ }
+fi
+
+ANY_CHANGED=0
+
+tmpfname=""
+
+#
+# Set up a trap handler to make sure that on exit, we remove our
+# tmpfile and un-stash the git environment (if appropriate)
+#
+trap 'if [ -n "${tmpfname}" ]; then rm -f "${tmpfname}"; fi; restoregit' 0
+
+for fname in "${INPUTS[@]}"; do
+ note "Inspecting $fname..."
+ tmpfname="${fname}.$$.clang_fmt.tmp"
+ rm -f "${tmpfname}"
+ clang-format --style=file "${fname}" > "${tmpfname}"
+ "${SCRIPT_DIR}/codetool.py" "${tmpfname}"
+
+ changed=not_set
+
+ if [[ $DIFFMODE = 1 ]]; then
+ # If we're running diff for its output, we can also use it
+ # to compare the files.
+ if diff -u "${fname}" "${tmpfname}"; then
+ changed=0
+ else
+ changed=1
+ fi
+ else
+ # We aren't running diff, so we have to compare the files with cmp.
+ if cmp "${fname}" "${tmpfname}" >/dev/null 2>&1; then
+ changed=0
+ else
+ changed=1
+ fi
+ fi
+
+ if [[ $changed = 1 ]]; then
+ note "Found a change in $fname"
+ ANY_CHANGED=1
+
+ if [[ $CHANGEMODE = 1 ]]; then
+ mv "${tmpfname}" "${fname}"
+ fi
+ fi
+
+ rm -f "${tmpfname}"
+done
+
+exitcode=0
+
+if [[ $CHECKMODE = 1 ]]; then
+ if [[ $ANY_CHANGED = 1 ]]; then
+ note "Found at least one misformatted file; check failed"
+ exitcode=1
+ else
+ note "No changes found."
+ fi
+fi
+
+exit $exitcode
diff --git a/scripts/maint/codetool.py b/scripts/maint/codetool.py
index 725712c0cc..c6daf759f5 100755
--- a/scripts/maint/codetool.py
+++ b/scripts/maint/codetool.py
@@ -11,7 +11,7 @@
#
"""
- This program uses a set of plugable filters to inspect and transform
+ This program uses a set of pluggable filters to inspect and transform
our C code.
"""
diff --git a/scripts/maint/practracker/.enable_practracker_in_hooks b/scripts/maint/practracker/.enable_practracker_in_hooks
new file mode 100644
index 0000000000..a9e707f5da
--- /dev/null
+++ b/scripts/maint/practracker/.enable_practracker_in_hooks
@@ -0,0 +1 @@
+This file is present to tell our git hooks to run practracker on this branch.
diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt
index a84a434cd6..2a225e07ee 100644
--- a/scripts/maint/practracker/exceptions.txt
+++ b/scripts/maint/practracker/exceptions.txt
@@ -34,10 +34,10 @@
# Remember: It is better to fix the problem than to add a new exception!
problem file-size /src/app/config/config.c 7525
-problem include-count /src/app/config/config.c 80
+problem include-count /src/app/config/config.c 81
problem function-size /src/app/config/config.c:options_act() 381
problem function-size /src/app/config/config.c:options_validate_cb() 794
-problem function-size /src/app/config/config.c:options_init_from_torrc() 192
+problem function-size /src/app/config/config.c:options_init_from_torrc() 139
problem function-size /src/app/config/config.c:options_init_from_string() 103
problem function-size /src/app/config/config.c:options_init_logs() 125
problem function-size /src/app/config/config.c:parse_bridge_line() 104
@@ -46,11 +46,11 @@ problem function-size /src/app/config/config.c:parse_dir_authority_line() 150
problem function-size /src/app/config/config.c:parse_dir_fallback_line() 101
problem function-size /src/app/config/config.c:port_parse_config() 435
problem function-size /src/app/config/config.c:parse_ports() 132
-problem function-size /src/app/config/resolve_addr.c:resolve_my_address() 191
-problem file-size /src/app/config/or_options_st.h 1050
-problem include-count /src/app/main/main.c 68
+problem function-size /src/app/config/resolve_addr.c:resolve_my_address_v4() 197
+problem file-size /src/app/config/or_options_st.h 1072
+problem include-count /src/app/main/main.c 71
problem function-size /src/app/main/main.c:dumpstats() 102
-problem function-size /src/app/main/main.c:tor_init() 101
+problem function-size /src/app/main/main.c:tor_init() 109
problem function-size /src/app/main/main.c:sandbox_init_filter() 291
problem function-size /src/app/main/main.c:run_tor_main_loop() 105
problem function-size /src/app/main/ntmain.c:nt_service_install() 126
@@ -96,7 +96,7 @@ problem function-size /src/core/or/channeltls.c:channel_tls_process_authenticate
problem dependency-violation /src/core/or/channeltls.c 11
problem include-count /src/core/or/circuitbuild.c 53
problem function-size /src/core/or/circuitbuild.c:get_unique_circ_id_by_chan() 128
-problem function-size /src/core/or/circuitbuild.c:choose_good_exit_server_general() 206
+problem function-size /src/core/or/circuitbuild.c:choose_good_exit_server_general() 196
problem dependency-violation /src/core/or/circuitbuild.c 25
problem include-count /src/core/or/circuitlist.c 55
problem function-size /src/core/or/circuitlist.c:HT_PROTOTYPE() 109
@@ -108,16 +108,16 @@ problem dependency-violation /src/core/or/circuitlist.h 1
problem function-size /src/core/or/circuitmux.c:circuitmux_set_policy() 109
problem function-size /src/core/or/circuitmux.c:circuitmux_attach_circuit() 113
problem dependency-violation /src/core/or/circuitmux_ewma.c 2
-problem file-size /src/core/or/circuitpadding.c 3101
+problem file-size /src/core/or/circuitpadding.c 3183
problem function-size /src/core/or/circuitpadding.c:circpad_machine_schedule_padding() 113
problem dependency-violation /src/core/or/circuitpadding.c 6
-problem file-size /src/core/or/circuitpadding.h 813
+problem file-size /src/core/or/circuitpadding.h 832
problem function-size /src/core/or/circuitpadding_machines.c:circpad_machine_relay_hide_intro_circuits() 103
problem function-size /src/core/or/circuitpadding_machines.c:circpad_machine_client_hide_rend_circuits() 112
problem dependency-violation /src/core/or/circuitpadding_machines.c 1
problem function-size /src/core/or/circuitstats.c:circuit_build_times_parse_state() 123
problem dependency-violation /src/core/or/circuitstats.c 11
-problem file-size /src/core/or/circuituse.c 3195
+problem file-size /src/core/or/circuituse.c 3250
problem function-size /src/core/or/circuituse.c:circuit_is_acceptable() 128
problem function-size /src/core/or/circuituse.c:circuit_expire_building() 389
problem function-size /src/core/or/circuituse.c:circuit_log_ancient_one_hop_circuits() 126
@@ -146,8 +146,9 @@ problem function-size /src/core/or/connection_or.c:connection_or_group_set_badne
problem function-size /src/core/or/connection_or.c:connection_or_client_learned_peer_id() 142
problem dependency-violation /src/core/or/connection_or.c 21
problem dependency-violation /src/core/or/dos.c 6
+problem dependency-violation /src/core/or/extendinfo.c 6
problem dependency-violation /src/core/or/onion.c 2
-problem file-size /src/core/or/or.h 1105
+problem file-size /src/core/or/or.h 1150
problem include-count /src/core/or/or.h 48
problem dependency-violation /src/core/or/or.h 1
problem dependency-violation /src/core/or/or_periodic.c 1
@@ -189,7 +190,7 @@ problem function-size /src/feature/client/entrynodes.c:entry_guard_parse_from_st
problem file-size /src/feature/client/entrynodes.h 700
problem function-size /src/feature/client/transports.c:handle_proxy_line() 108
problem function-size /src/feature/client/transports.c:parse_method_line_helper() 110
-problem function-size /src/feature/client/transports.c:create_managed_proxy_environment() 111
+problem function-size /src/feature/client/transports.c:create_managed_proxy_environment() 140
problem function-size /src/feature/control/control.c:connection_control_process_inbuf() 113
problem function-size /src/feature/control/control_auth.c:handle_control_authenticate() 186
problem function-size /src/feature/control/control_cmd.c:handle_control_extendcircuit() 150
@@ -199,9 +200,9 @@ problem function-size /src/feature/control/control_events.c:control_event_stream
problem include-count /src/feature/control/control_getinfo.c 56
problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_misc() 108
problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_dir() 297
-problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_events() 234
+problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_events() 237
problem function-size /src/feature/dirauth/bwauth.c:dirserv_read_measured_bandwidths() 121
-problem file-size /src/feature/dirauth/dirvote.c 4734
+problem file-size /src/feature/dirauth/dirvote.c 4900
problem include-count /src/feature/dirauth/dirvote.c 55
problem function-size /src/feature/dirauth/dirvote.c:format_networkstatus_vote() 230
problem function-size /src/feature/dirauth/dirvote.c:networkstatus_compute_bw_weights_v10() 233
@@ -256,11 +257,11 @@ problem function-size /src/feature/nodelist/microdesc.c:microdesc_cache_rebuild(
problem include-count /src/feature/nodelist/networkstatus.c 65
problem function-size /src/feature/nodelist/networkstatus.c:networkstatus_check_consensus_signature() 175
problem function-size /src/feature/nodelist/networkstatus.c:networkstatus_set_current_consensus() 289
-problem function-size /src/feature/nodelist/node_select.c:router_pick_directory_server_impl() 122
+problem function-size /src/feature/nodelist/node_select.c:router_pick_directory_server_impl() 126
problem function-size /src/feature/nodelist/node_select.c:compute_weighted_bandwidths() 204
-problem function-size /src/feature/nodelist/node_select.c:router_pick_trusteddirserver_impl() 112
+problem function-size /src/feature/nodelist/node_select.c:router_pick_trusteddirserver_impl() 116
problem function-size /src/feature/nodelist/nodelist.c:compute_frac_paths_available() 190
-problem file-size /src/feature/nodelist/routerlist.c 3247
+problem file-size /src/feature/nodelist/routerlist.c 3350
problem function-size /src/feature/nodelist/routerlist.c:router_rebuild_store() 148
problem function-size /src/feature/nodelist/routerlist.c:router_add_to_routerlist() 168
problem function-size /src/feature/nodelist/routerlist.c:routerlist_remove_old_routers() 121
@@ -324,3 +325,5 @@ problem function-size /src/tools/tor-gencert.c:parse_commandline() 111
problem function-size /src/tools/tor-resolve.c:build_socks5_resolve_request() 102
problem function-size /src/tools/tor-resolve.c:do_resolve() 171
problem function-size /src/tools/tor-resolve.c:main() 112
+problem dependency-violation /src/core/or/trace_probes_circuit.c 1
+problem dependency-violation /src/core/or/trace_probes_circuit.h 1
diff --git a/scripts/maint/rename_c_identifier.py b/scripts/maint/rename_c_identifier.py
index 77802e10f3..8b286c1a28 100755
--- a/scripts/maint/rename_c_identifier.py
+++ b/scripts/maint/rename_c_identifier.py
@@ -153,7 +153,7 @@ This is an automated commit, generated by this command:
if no_verify:
msg += """
It was generated with --no-verify, so it probably breaks some commit hooks.
-The commiter should be sure to fix them up in a subsequent commit.
+The committer should be sure to fix them up in a subsequent commit.
"""
return msg
@@ -239,7 +239,7 @@ def main(argv):
print("I require an even number of identifiers.", file=sys.stderr)
return 1
- if any_uncommitted_changes():
+ if args.commit and any_uncommitted_changes():
print("Uncommitted changes found. Not running.", file=sys.stderr)
return 1
7apu2bq3rhoylwmucxizk54afw5jyda7pho4j5zdcqpwkufke7zdzwad.onion