diff options
Diffstat (limited to 'scripts')
39 files changed, 1477 insertions, 3522 deletions
diff --git a/scripts/README b/scripts/README index 02faabe06b..9cd6e74ac7 100644 --- a/scripts/README +++ b/scripts/README @@ -30,6 +30,12 @@ orconfig.h files. Testing scripts --------------- +test/chutney-git-bisect.sh -- a git bisect run script that bisects using +chutney. The script builds tor and tor-gencert, then runs chutney. The script +takes optional arguments for out-of-tree builds, and specific chutney network +flavours. You should copy this script before using it with git bisect, so that +it doesn't change (or disappear) during bisection. + test/cov-blame -- Mash up the results of gcov with git blame. Mainly useful to find out who has been writing untested code. diff --git a/scripts/coccinelle/ceil_div.cocci b/scripts/coccinelle/ceil_div.cocci new file mode 100644 index 0000000000..00843e82c3 --- /dev/null +++ b/scripts/coccinelle/ceil_div.cocci @@ -0,0 +1,6 @@ +@@ +expression n, d; +@@ + +- (((n) + (d) - 1) / (d)) ++ CEIL_DIV(n, d) diff --git a/scripts/coccinelle/test-operator-cleanup b/scripts/coccinelle/test-operator-cleanup new file mode 100755 index 0000000000..e7822542a4 --- /dev/null +++ b/scripts/coccinelle/test-operator-cleanup @@ -0,0 +1,11 @@ +#!/usr/bin/perl -w -p -i + +next if m#^ */\*# or m#^ *\* #; + +s/<([,)])/OP_LT$1/; +s/(?<=[\s,])>([,)])/OP_GT$1/; +#s/>([,)])/OP_GT$1/; +s/==([,)])/OP_EQ$1/; +s/>=([,)])/OP_GE$1/; +s/<=([,)])/OP_LE$1/; +s/!=([,)])/OP_NE$1/; diff --git a/scripts/coccinelle/test_assert_int.cocci b/scripts/coccinelle/test_assert_int.cocci new file mode 100644 index 0000000000..80e86b4f3c --- /dev/null +++ b/scripts/coccinelle/test_assert_int.cocci @@ -0,0 +1,49 @@ +@@ +int e; +constant c; +@@ + +( +- tt_assert(e == c) ++ tt_int_op(e, OP_EQ, c) +| +- tt_assert(e != c) ++ tt_int_op(e, OP_NE, c) +| +- tt_assert(e < c) ++ tt_int_op(e, OP_LT, c) +| +- tt_assert(e <= c) ++ tt_int_op(e, OP_LE, c) +| +- tt_assert(e > c) ++ tt_int_op(e, OP_GT, c) +| +- tt_assert(e >= c) ++ tt_int_op(e, OP_GE, c) +) + +@@ +unsigned int e; +constant c; +@@ + +( +- tt_assert(e == c) ++ tt_uint_op(e, OP_EQ, c) +| +- tt_assert(e != c) ++ tt_uint_op(e, OP_NE, c) +| +- tt_assert(e < c) ++ tt_uint_op(e, OP_LT, c) +| +- tt_assert(e <= c) ++ tt_uint_op(e, OP_LE, c) +| +- tt_assert(e > c) ++ tt_uint_op(e, OP_GT, c) +| +- tt_assert(e >= c) ++ tt_uint_op(e, OP_GE, c) +) diff --git a/scripts/coccinelle/test_assert_null.cocci b/scripts/coccinelle/test_assert_null.cocci new file mode 100644 index 0000000000..3d66e1ee0b --- /dev/null +++ b/scripts/coccinelle/test_assert_null.cocci @@ -0,0 +1,11 @@ +@@ +expression * e; +@@ + +( +- tt_assert(e != NULL) ++ tt_ptr_op(e, OP_NE, NULL) +| +- tt_assert(e == NULL) ++ tt_ptr_op(e, OP_EQ, NULL) +) diff --git a/scripts/coccinelle/test_assert_zero.cocci b/scripts/coccinelle/test_assert_zero.cocci new file mode 100644 index 0000000000..09feaa5fb4 --- /dev/null +++ b/scripts/coccinelle/test_assert_zero.cocci @@ -0,0 +1,5 @@ +@@ +@@ + +- tt_assert(0) ++ tt_abort() diff --git a/scripts/codegen/fuzzing_include_am.py b/scripts/codegen/fuzzing_include_am.py new file mode 100755 index 0000000000..a944584453 --- /dev/null +++ b/scripts/codegen/fuzzing_include_am.py @@ -0,0 +1,155 @@ +#!/usr/bin/python + +FUZZERS = """ + consensus + descriptor + diff + diff-apply + extrainfo + hsdescv2 + hsdescv3 + http + http-connect + iptsv2 + microdesc + socks + strops + vrs +""" + + +PREAMBLE = r""" +FUZZING_CPPFLAGS = \ + $(src_test_AM_CPPFLAGS) $(TEST_CPPFLAGS) +FUZZING_CFLAGS = \ + $(AM_CFLAGS) $(TEST_CFLAGS) +FUZZING_LDFLAG = \ + @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) @TOR_LDFLAGS_libevent@ +FUZZING_LIBS = \ + $(TOR_INTERNAL_TESTING_LIBS) \ + $(rust_ldadd) \ + @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ + @TOR_LIBEVENT_LIBS@ $(TOR_LIBS_CRYPTLIB) \ + @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ @CURVE25519_LIBS@ \ + @TOR_SYSTEMD_LIBS@ \ + @TOR_LZMA_LIBS@ \ + @TOR_ZSTD_LIBS@ + +oss-fuzz-prereqs: \ + $(TOR_INTERNAL_TESTING_LIBS) + +noinst_HEADERS += \ + src/test/fuzz/fuzzing.h + +LIBFUZZER = -lFuzzer +LIBFUZZER_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ +LIBFUZZER_CFLAGS = $(FUZZING_CFLAGS) +LIBFUZZER_LDFLAG = $(FUZZING_LDFLAG) +LIBFUZZER_LIBS = $(FUZZING_LIBS) $(LIBFUZZER) -lstdc++ + +LIBOSS_FUZZ_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ +LIBOSS_FUZZ_CFLAGS = $(FUZZING_CFLAGS) +""" + +POSTAMBLE = r""" +noinst_PROGRAMS += $(FUZZERS) $(LIBFUZZER_FUZZERS) +noinst_LIBRARIES += $(OSS_FUZZ_FUZZERS) +oss-fuzz-fuzzers: oss-fuzz-prereqs $(OSS_FUZZ_FUZZERS) +fuzzers: $(FUZZERS) $(LIBFUZZER_FUZZERS) + +test-fuzz-corpora: $(FUZZERS) + $(top_srcdir)/src/test/fuzz_static_testcases.sh +""" + +########### No user serviceable parts will follow. + +PREAMBLE = PREAMBLE.strip() +POSTAMBLE = POSTAMBLE.strip() # If I use it, it's a word! +FUZZERS = FUZZERS.split() +FUZZERS.sort() + +WARNING = """ +# This file was generated by fuzzing_include_am.py; do not hand-edit unless +# you enjoy having your changes erased. +""".strip() + +print(WARNING) + +print(PREAMBLE) + +print("\n# ===== AFL fuzzers") + +def get_id_name(s): + return s.replace("-", "_") + +for fuzzer in FUZZERS: + idname = get_id_name(fuzzer) + print("""\ +if UNITTESTS_ENABLED +src_test_fuzz_fuzz_{name}_SOURCES = \\ + src/test/fuzz/fuzzing_common.c \\ + src/test/fuzz/fuzz_{name}.c +src_test_fuzz_fuzz_{name}_CPPFLAGS = $(FUZZING_CPPFLAGS) +src_test_fuzz_fuzz_{name}_CFLAGS = $(FUZZING_CFLAGS) +src_test_fuzz_fuzz_{name}_LDFLAGS = $(FUZZING_LDFLAG) +src_test_fuzz_fuzz_{name}_LDADD = $(FUZZING_LIBS) +endif +""".format(name=idname)) + +print("if UNITTESTS_ENABLED") +print("FUZZERS = \\") +print(" \\\n".join("\tsrc/test/fuzz/fuzz-{name}".format(name=fuzzer) + for fuzzer in FUZZERS)) +print("endif") + +print("\n# ===== libfuzzer") +print("\nif LIBFUZZER_ENABLED") + +for fuzzer in FUZZERS: + idname = get_id_name(fuzzer) + print("""\ +if UNITTESTS_ENABLED +src_test_fuzz_lf_fuzz_{name}_SOURCES = \\ + $(src_test_fuzz_fuzz_{name}_SOURCES) +src_test_fuzz_lf_fuzz_{name}_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) +src_test_fuzz_lf_fuzz_{name}_CFLAGS = $(LIBFUZZER_CFLAGS) +src_test_fuzz_lf_fuzz_{name}_LDFLAGS = $(LIBFUZZER_LDFLAG) +src_test_fuzz_lf_fuzz_{name}_LDADD = $(LIBFUZZER_LIBS) +endif +""".format(name=idname)) + +print("LIBFUZZER_FUZZERS = \\") +print(" \\\n".join("\tsrc/test/fuzz/lf-fuzz-{name}".format(name=fuzzer) + for fuzzer in FUZZERS)) + +print(""" +else +LIBFUZZER_FUZZERS = +endif""") + +print("\n# ===== oss-fuzz\n") +print("if OSS_FUZZ_ENABLED") + +for fuzzer in FUZZERS: + idname = get_id_name(fuzzer) + print("""\ +if UNITTESTS_ENABLED +src_test_fuzz_liboss_fuzz_{name}_a_SOURCES = \\ + $(src_test_fuzz_fuzz_{name}_SOURCES) +src_test_fuzz_liboss_fuzz_{name}_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) +src_test_fuzz_liboss_fuzz_{name}_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif +""".format(name=idname)) + +print("OSS_FUZZ_FUZZERS = \\") +print(" \\\n".join("\tsrc/test/fuzz/liboss-fuzz-{name}.a".format(name=fuzzer) + for fuzzer in FUZZERS)) + +print(""" +else +OSS_FUZZ_FUZZERS = +endif""") + +print("") + +print(POSTAMBLE) diff --git a/scripts/codegen/gen_server_ciphers.py b/scripts/codegen/gen_server_ciphers.py index 0dca8a6734..5d326f8b9e 100755 --- a/scripts/codegen/gen_server_ciphers.py +++ b/scripts/codegen/gen_server_ciphers.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 2014-2015, The Tor Project, Inc +# Copyright 2014-2019, The Tor Project, Inc # See LICENSE for licensing information # This script parses openssl headers to find ciphersuite names, determines @@ -13,13 +13,13 @@ import sys EPHEMERAL_INDICATORS = [ "_EDH_", "_DHE_", "_ECDHE_" ] BAD_STUFF = [ "_DES_40_", "MD5", "_RC4_", "_DES_64_", - "_SEED_", "_CAMELLIA_", "_NULL" ] + "_SEED_", "_CAMELLIA_", "_NULL", + "_CCM_8", "_DES_", ] # these never get #ifdeffed. MANDATORY = [ "TLS1_TXT_DHE_RSA_WITH_AES_256_SHA", "TLS1_TXT_DHE_RSA_WITH_AES_128_SHA", - "SSL3_TXT_EDH_RSA_DES_192_CBC3_SHA", ] def find_ciphers(filename): @@ -48,15 +48,23 @@ def usable_cipher(ciph): # All fields we sort on, in order of priority. FIELDS = [ 'cipher', 'fwsec', 'mode', 'digest', 'bitlength' ] # Map from sorted fields to recognized value in descending order of goodness -FIELD_VALS = { 'cipher' : [ 'AES', 'DES'], +FIELD_VALS = { 'cipher' : [ 'AES', 'CHACHA20' ], 'fwsec' : [ 'ECDHE', 'DHE' ], - 'mode' : [ 'GCM', 'CBC' ], - 'digest' : [ 'SHA384', 'SHA256', 'SHA' ], + 'mode' : [ 'POLY1305', 'GCM', 'CCM', 'CBC', ], + 'digest' : [ 'n/a', 'SHA384', 'SHA256', 'SHA', ], 'bitlength' : [ '256', '128', '192' ], } class Ciphersuite(object): def __init__(self, name, fwsec, cipher, bitlength, mode, digest): + if fwsec == 'EDH': + fwsec = 'DHE' + + if mode in [ '_CBC3', '_CBC', '' ]: + mode = 'CBC' + elif mode == '_GCM': + mode = 'GCM' + self.name = name self.fwsec = fwsec self.cipher = cipher @@ -74,42 +82,50 @@ class Ciphersuite(object): def parse_cipher(ciph): m = re.match('(?:TLS1|SSL3)_TXT_(EDH|DHE|ECDHE)_RSA(?:_WITH)?_(AES|DES)_(256|128|192)(|_CBC|_CBC3|_GCM)_(SHA|SHA256|SHA384)$', ciph) - if not m: - print "/* Couldn't parse %s ! */"%ciph - return None + if m: + fwsec, cipher, bits, mode, digest = m.groups() + return Ciphersuite(ciph, fwsec, cipher, bits, mode, digest) - fwsec, cipher, bits, mode, digest = m.groups() - if fwsec == 'EDH': - fwsec = 'DHE' + m = re.match('(?:TLS1|SSL3)_TXT_(EDH|DHE|ECDHE)_RSA(?:_WITH)?_(AES|DES)_(256|128|192)_CCM', ciph) + if m: + fwsec, cipher, bits = m.groups() + return Ciphersuite(ciph, fwsec, cipher, bits, "CCM", "n/a") - if mode in [ '_CBC3', '_CBC', '' ]: - mode = 'CBC' - elif mode == '_GCM': - mode = 'GCM' + m = re.match('(?:TLS1|SSL3)_TXT_(EDH|DHE|ECDHE)_RSA(?:_WITH)?_CHACHA20_POLY1305', ciph) + if m: + fwsec, = m.groups() + return Ciphersuite(ciph, fwsec, "CHACHA20", "256", "POLY1305", "n/a") + + print "/* Couldn't parse %s ! */"%ciph + return None - return Ciphersuite(ciph, fwsec, cipher, bits, mode, digest) ALL_CIPHERS = [] for fname in sys.argv[1:]: - ALL_CIPHERS += (parse_cipher(c) - for c in find_ciphers(fname) - if usable_cipher(c) ) + for c in find_ciphers(fname): + if usable_cipher(c): + parsed = parse_cipher(c) + if parsed != None: + ALL_CIPHERS.append(parsed) ALL_CIPHERS.sort(key=Ciphersuite.sort_key) +indent = " "*7 + for c in ALL_CIPHERS: if c is ALL_CIPHERS[-1]: - colon = ';' + colon = '' else: colon = ' ":"' if c.name in MANDATORY: - print " /* Required */" - print ' %s%s'%(c.name,colon) + print "%s/* Required */"%indent + print '%s%s%s'%(indent,c.name,colon) else: print "#ifdef %s"%c.name - print ' %s%s'%(c.name,colon) + print '%s%s%s'%(indent,c.name,colon) print "#endif" +print '%s;'%indent diff --git a/scripts/codegen/get_mozilla_ciphers.py b/scripts/codegen/get_mozilla_ciphers.py index e673ec7dc6..f23f2f1e6f 100644..100755 --- a/scripts/codegen/get_mozilla_ciphers.py +++ b/scripts/codegen/get_mozilla_ciphers.py @@ -1,6 +1,6 @@ #!/usr/bin/python # coding=utf-8 -# Copyright 2011-2015, The Tor Project, Inc +# Copyright 2011-2019, The Tor Project, Inc # original version by Arturo Filastò # See LICENSE for licensing information @@ -127,9 +127,9 @@ for k, v in enabled_ciphers.items(): #oSSLinclude = ('/usr/include/openssl/ssl3.h', '/usr/include/openssl/ssl.h', # '/usr/include/openssl/ssl2.h', '/usr/include/openssl/ssl23.h', # '/usr/include/openssl/tls1.h') -oSSLinclude = ('ssl/ssl3.h', 'ssl/ssl.h', - 'ssl/ssl2.h', 'ssl/ssl23.h', - 'ssl/tls1.h') +oSSLinclude = ['ssl3.h', 'ssl.h' + 'ssl2.h', 'ssl23.h', + 'tls1.h'] ##### # This reads the hex code for the ciphers that are used by firefox. @@ -155,9 +155,12 @@ for x in used_ciphers: openssl_macro_by_hex = {} all_openssl_macros = {} for fl in oSSLinclude: - fp = open(ossl(fl), 'r') + fname = ossl("include/openssl/"+fl) + if not os.path.exists(fname): + continue + fp = open(fname, 'r') for line in fp.readlines(): - m = re.match('#define\s+(\S+)\s+(\S+)', line) + m = re.match('# *define\s+(\S+)\s+(\S+)', line) if m: value,key = m.groups() if key.startswith('0x') and "_CK_" in value: diff --git a/scripts/codegen/makedesc.py b/scripts/codegen/makedesc.py index d4ba21efae..efca4dda9a 100644 --- a/scripts/codegen/makedesc.py +++ b/scripts/codegen/makedesc.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 2014-2015, The Tor Project, Inc. +# Copyright 2014-2019, The Tor Project, Inc. # See LICENSE for license information # This is a kludgey python script that uses ctypes and openssl to sign diff --git a/scripts/codegen/run_trunnel.sh b/scripts/codegen/run_trunnel.sh index d2669931e9..645b3c2158 100755 --- a/scripts/codegen/run_trunnel.sh +++ b/scripts/codegen/run_trunnel.sh @@ -5,7 +5,11 @@ if test "x$TRUNNEL_PATH" != "x"; then export PYTHONPATH fi -python -m trunnel --require-version=1.4 ./src/trunnel/*.trunnel +OPTIONS="--require-version=1.5.1" -python -m trunnel --require-version=1.4 --write-c-files --target-dir=./src/ext/trunnel/ +# Get all .trunnel files recursively from that directory so we can support +# multiple sub-directories. +find ./src/trunnel/ -name '*.trunnel' -exec python -m trunnel ${OPTIONS} {} \; + +python -m trunnel ${OPTIONS} --write-c-files --target-dir=./src/ext/trunnel/ diff --git a/scripts/maint/analyze_callgraph.py b/scripts/maint/analyze_callgraph.py deleted file mode 100755 index 8ce5827f07..0000000000 --- a/scripts/maint/analyze_callgraph.py +++ /dev/null @@ -1,259 +0,0 @@ -#!/usr/bin/python - -import re -import sys -import copy -import cPickle -import os - -class Parser: - def __init__(self): - self.calls = {} - self.definedIn = {} - - def enter_func(self, name): - if self.infunc and not self.extern and self.calledfns: - if self.infunc in self.definedIn: - #print "{}: {} or {}?".format( - # self.infunc, self.definedIn[self.infunc], self.module) - self.definedIn[self.infunc] = 'nil' - else: - self.definedIn[self.infunc] = self.module - self.calls.setdefault(self.infunc, set()).update( self.calledfns ) - - self.calledfns = set() - self.infunc = name - self.extern = False - - def parse_callgraph_file(self, inp, module): - self.infunc = None - self.extern = False - self.calledfns = set() - self.module = module - - for line in inp: - m = re.match(r"Call graph node for function: '([^']+)'", line) - if m: - self.enter_func(m.group(1)) - continue - m = re.match(r" CS<[^>]+> calls external node", line) - if m: - self.extern = True - m = re.match(r" CS<[^>]+> calls function '([^']+)'", line) - if m: - self.calledfns.add(m.group(1)) - self.enter_func(None) - - def extract_callgraph(self): - c = self.calls - self.calls = {} - return c - - -def transitive_closure(g): - passno = 0 - changed = True - g = copy.deepcopy(g) - import random - while changed: - passno += 1 - changed = False - keys = g.keys() - idx = 0 - for k in keys: - idx += 1 - print "Pass %d/?: %d/%d\r" %(passno, idx, len(keys)), - sys.stdout.flush() - newset = g[k].copy() - for fn in g[k]: - newset.update(g.get(fn, set())) - if len(newset) != len(g[k]): - g[k].update( newset ) - changed = True - - print - - return g - -def strongly_connected_components(g): - # From https://en.wikipedia.org/wiki/Tarjan%27s_strongly_connected_components_algorithm, done stupidly. - index_of = {} - index = [ 0 ] - lowlink = {} - S = [] - onStack = set() - - all_sccs = [] - - def strongconnect(fn): - index_of[fn] = index[0] - lowlink[fn] = index[0] - index[0] += 1 - S.append(fn) - onStack.add(fn) - - for w in g.get(fn, []): - if w not in index_of: - strongconnect(w) - lowlink[fn] = min(lowlink[fn], lowlink[w]) - elif w in onStack: - lowlink[fn] = min(lowlink[fn], index_of[w]) - - if lowlink[fn] == index_of[fn]: - this_scc = [] - all_sccs.append(this_scc) - while True: - w = S.pop() - onStack.remove(w) - this_scc.append(w) - if w == fn: - break - - for v in g.keys(): - if v not in index_of: - strongconnect(v) - - return all_sccs - -def biggest_component(sccs): - return max(len(c) for c in sccs) - -def connection_bottlenecks(callgraph): - - callers = {} - for fn in callgraph: - for fn2 in callgraph[fn]: - callers.setdefault(fn2, set()).add(fn) - - components = strongly_connected_components(callgraph) - components.sort(key=len) - big_component_fns = components[-1] - size = len(big_component_fns) - - function_bottlenecks = fn_results = [] - - total = len(big_component_fns) - idx = 0 - for fn in big_component_fns: - idx += 1 - print "Pass 1/3: %d/%d\r"%(idx, total), - sys.stdout.flush() - cg2 = copy.deepcopy(callgraph) - del cg2[fn] - - fn_results.append( (size - biggest_component(strongly_connected_components(cg2)), fn) ) - - print - bcf_set = set(big_component_fns) - - call_bottlenecks = fn_results = [] - result_set = set() - total = len(big_component_fns) - idx = 0 - for fn in big_component_fns: - fn_callers = callers[fn].intersection(bcf_set) - idx += 1 - if len(fn_callers) != 1: - continue - - print "Pass 2/3: %d/%d\r"%(idx, total), - sys.stdout.flush() - - caller = fn_callers.pop() - assert len(fn_callers) == 0 - cg2 = copy.deepcopy(callgraph) - cg2[caller].remove(fn) - - fn_results.append( (size - biggest_component(strongly_connected_components(cg2)), fn, "called by", caller) ) - result_set.add( (caller, fn) ) - - print - - total = len(big_component_fns) - idx = 0 - for fn in big_component_fns: - fn_calls = callgraph[fn].intersection(bcf_set) - idx += 1 - if len(fn_calls) != 1: - continue - - print "Pass 3/3: %d/%d\r"%(idx, total), - sys.stdout.flush() - - callee = fn_calls.pop() - if (fn, callee) in result_set: - continue - - assert len(fn_calls) == 0 - cg2 = copy.deepcopy(callgraph) - cg2[fn].remove(callee) - - fn_results.append( (size - biggest_component(strongly_connected_components(cg2)), callee, "called by", fn) ) - - print - - - return (function_bottlenecks, call_bottlenecks) - -if __name__ == '__main__': - p = Parser() - for fname in sys.argv[1:]: - modname = re.sub(r'.*/', '', fname).replace('.callgraph', '.c') - with open(fname, 'r') as f: - p.parse_callgraph_file(f, modname) - - sys.stdout.flush() - - print "Building callgraph" - callgraph = p.extract_callgraph() - inModule = p.definedIn - - print "Deriving module callgraph" - modCallgraph = {} - for fn in callgraph: - fnMod = inModule[fn] - for called in callgraph[fn]: - try: - calledMod = inModule[called] - except KeyError: - continue - modCallgraph.setdefault(fnMod, set()).add(calledMod) - del modCallgraph['nil'] - - print "Finding strongly connected components" - sccs = strongly_connected_components(callgraph) - - print "Finding the transitive closure of the callgraph.." - closure = transitive_closure(callgraph) - - print "Finding bottlenecks..." - bottlenecks = connection_bottlenecks(callgraph) - - print "Finding module SCCs" - modSCCS = strongly_connected_components(modCallgraph) - - print "Finding module TC" - modTC = transitive_closure(modCallgraph) - - print "Finding module bottlenecks" - modB = connection_bottlenecks(modCallgraph) - - data = { - 'callgraph' : callgraph, - 'sccs' : sccs, - 'closure' : closure, - 'bottlenecks' : bottlenecks, - 'modules' : p.definedIn, - 'modItems' : { - 'callgraph' : modCallgraph, - 'sccs' : modSCCS, - 'closure' : modTC, - 'bottlenecks' : modB, - } - } - - with open('callgraph.pkl', 'w') as f: - cPickle.dump(data, f) - - - diff --git a/scripts/maint/annotate_ifdef_directives b/scripts/maint/annotate_ifdef_directives new file mode 100755 index 0000000000..ca267a865e --- /dev/null +++ b/scripts/maint/annotate_ifdef_directives @@ -0,0 +1,74 @@ +#!/usr/bin/python +# Copyright (c) 2017-2019, The Tor Project, Inc. +# See LICENSE for licensing information + +import re + +LINE_OBVIOUSNESS_LIMIT = 4 + +class Problem(Exception): + pass + +def uncomment(s): + s = re.sub(r'//.*','',s) + s = re.sub(r'/\*.*','',s) + return s.strip() + +def translate(f_in, f_out): + whole_file = [] + stack = [] + cur_level = whole_file + lineno = 0 + for line in f_in: + lineno += 1 + m = re.match(r'\s*#\s*(if|ifdef|ifndef|else|endif|elif)\b\s*(.*)', + line) + if not m: + f_out.write(line) + continue + command,rest = m.groups() + if command in ("if", "ifdef", "ifndef"): + # The #if directive pushes us one level lower on the stack. + if command == 'ifdef': + rest = "defined(%s)"%uncomment(rest) + elif command == 'ifndef': + rest = "!defined(%s)"%uncomment(rest) + elif rest.endswith("\\"): + rest = rest[:-1]+"..." + + rest = uncomment(rest) + + new_level = [ (command, rest, lineno) ] + stack.append(cur_level) + cur_level = new_level + f_out.write(line) + elif command in ("else", "elif"): + if len(cur_level) == 0 or cur_level[-1][0] == 'else': + raise Problem("Unexpected #%s on %d"% (command,lineno)) + if (len(cur_level) == 1 and command == 'else' and + lineno > cur_level[0][2] + LINE_OBVIOUSNESS_LIMIT): + f_out.write("#else /* !(%s) */\n"%cur_level[0][1]) + else: + f_out.write(line) + cur_level.append((command, rest, lineno)) + else: + assert command == 'endif' + if len(stack) == 0: + raise Problem("Unmatched #%s on %s"% (command,lineno)) + if lineno <= cur_level[0][2] + LINE_OBVIOUSNESS_LIMIT: + f_out.write(line) + elif len(cur_level) == 1 or ( + len(cur_level) == 2 and cur_level[1][0] == 'else'): + f_out.write("#endif /* %s */\n"%cur_level[0][1]) + else: + f_out.write("#endif /* %s || ... */\n"%cur_level[0][1]) + cur_level = stack.pop() + if len(stack) or cur_level != whole_file: + raise Problem("Missing #endif") + +import sys,os +for fn in sys.argv[1:]: + with open(fn+"_OUT", 'w') as output_file: + translate(open(fn, 'r'), output_file) + os.rename(fn+"_OUT", fn) + diff --git a/scripts/maint/checkIncludes.py b/scripts/maint/checkIncludes.py new file mode 100755 index 0000000000..3afd9bbebe --- /dev/null +++ b/scripts/maint/checkIncludes.py @@ -0,0 +1,181 @@ +#!/usr/bin/python3 +# Copyright 2018 The Tor Project, Inc. See LICENSE file for licensing info. + +"""This script looks through all the directories for files matching *.c or + *.h, and checks their #include directives to make sure that only "permitted" + headers are included. + + Any #include directives with angle brackets (like #include <stdio.h>) are + ignored -- only directives with quotes (like #include "foo.h") are + considered. + + To decide what includes are permitted, this script looks at a .may_include + file in each directory. This file contains empty lines, #-prefixed + comments, filenames (like "lib/foo/bar.h") and file globs (like lib/*/*.h) + for files that are permitted. +""" + + +from __future__ import print_function + +import fnmatch +import os +import re +import sys + +# Global: Have there been any errors? +trouble = False + +if sys.version_info[0] <= 2: + def open_file(fname): + return open(fname, 'r') +else: + def open_file(fname): + return open(fname, 'r', encoding='utf-8') + +def warn(msg): + print(msg, file=sys.stderr) + +def err(msg): + """ Declare that an error has happened, and remember that there has + been an error. """ + global trouble + trouble = True + print(msg, file=sys.stderr) + +def fname_is_c(fname): + """ Return true iff 'fname' is the name of a file that we should + search for possibly disallowed #include directives. """ + return fname.endswith(".h") or fname.endswith(".c") + +INCLUDE_PATTERN = re.compile(r'\s*#\s*include\s+"([^"]*)"') +RULES_FNAME = ".may_include" + +ALLOWED_PATTERNS = [ + re.compile(r'^.*\*\.(h|inc)$'), + re.compile(r'^.*/.*\.h$'), + re.compile(r'^ext/.*\.c$'), + re.compile(r'^orconfig.h$'), + re.compile(r'^micro-revision.i$'), +] + +def pattern_is_normal(s): + for p in ALLOWED_PATTERNS: + if p.match(s): + return True + return False + +class Rules(object): + """ A 'Rules' object is the parsed version of a .may_include file. """ + def __init__(self, dirpath): + self.dirpath = dirpath + if dirpath.startswith("src/"): + self.incpath = dirpath[4:] + else: + self.incpath = dirpath + self.patterns = [] + self.usedPatterns = set() + + def addPattern(self, pattern): + if not pattern_is_normal(pattern): + warn("Unusual pattern {} in {}".format(pattern, self.dirpath)) + self.patterns.append(pattern) + + def includeOk(self, path): + for pattern in self.patterns: + if fnmatch.fnmatchcase(path, pattern): + self.usedPatterns.add(pattern) + return True + return False + + def applyToLines(self, lines, context=""): + lineno = 0 + for line in lines: + lineno += 1 + m = INCLUDE_PATTERN.match(line) + if m: + include = m.group(1) + if not self.includeOk(include): + err("Forbidden include of {} on line {}{}".format( + include, lineno, context)) + + def applyToFile(self, fname): + with open_file(fname) as f: + #print(fname) + self.applyToLines(iter(f), " of {}".format(fname)) + + def noteUnusedRules(self): + for p in self.patterns: + if p not in self.usedPatterns: + print("Pattern {} in {} was never used.".format(p, self.dirpath)) + + def getAllowedDirectories(self): + allowed = [] + for p in self.patterns: + m = re.match(r'^(.*)/\*\.(h|inc)$', p) + if m: + allowed.append(m.group(1)) + continue + m = re.match(r'^(.*)/[^/]*$', p) + if m: + allowed.append(m.group(1)) + continue + + return allowed + +def load_include_rules(fname): + """ Read a rules file from 'fname', and return it as a Rules object. """ + result = Rules(os.path.split(fname)[0]) + with open_file(fname) as f: + for line in f: + line = line.strip() + if line.startswith("#") or not line: + continue + result.addPattern(line) + return result + +list_unused = False +log_sorted_levels = False + +uses_dirs = { } + +for dirpath, dirnames, fnames in os.walk("src"): + if ".may_include" in fnames: + rules = load_include_rules(os.path.join(dirpath, RULES_FNAME)) + for fname in fnames: + if fname_is_c(fname): + rules.applyToFile(os.path.join(dirpath,fname)) + if list_unused: + rules.noteUnusedRules() + + uses_dirs[rules.incpath] = rules.getAllowedDirectories() + +if trouble: + err( +"""To change which includes are allowed in a C file, edit the {} +files in its enclosing directory.""".format(RULES_FNAME)) + sys.exit(1) + +all_levels = [] + +n = 0 +while uses_dirs: + n += 0 + cur_level = [] + for k in list(uses_dirs): + uses_dirs[k] = [ d for d in uses_dirs[k] + if (d in uses_dirs and d != k)] + if uses_dirs[k] == []: + cur_level.append(k) + for k in cur_level: + del uses_dirs[k] + n += 1 + if cur_level and log_sorted_levels: + print(n, cur_level) + if n > 100: + break + +if uses_dirs: + print("There are circular .may_include dependencies in here somewhere:", + uses_dirs) + sys.exit(1) diff --git a/scripts/maint/checkOptionDocs.pl.in b/scripts/maint/checkOptionDocs.pl.in index 1f53adf099..6533c762c5 100644 --- a/scripts/maint/checkOptionDocs.pl.in +++ b/scripts/maint/checkOptionDocs.pl.in @@ -7,7 +7,7 @@ my %torrcSampleOptions = (); my %manPageOptions = (); # Load the canonical list as actually accepted by Tor. -open(F, "@abs_top_builddir@/src/or/tor --list-torrc-options |") or die; +open(F, "@abs_top_builddir@/src/app/tor --list-torrc-options |") or die; while (<F>) { next if m!\[notice\] Tor v0\.!; if (m!^([A-Za-z0-9_]+)!) { diff --git a/scripts/maint/checkSpace.pl b/scripts/maint/checkSpace.pl index 5af95a27e5..633b47e314 100755 --- a/scripts/maint/checkSpace.pl +++ b/scripts/maint/checkSpace.pl @@ -1,48 +1,68 @@ -#!/usr/bin/perl -w +#!/usr/bin/perl + +use strict; +use warnings; + +my $found = 0; +sub msg { + $found = 1; + print "$_[0]"; +} + +my $C = 0; if ($ARGV[0] =~ /^-/) { - $lang = shift @ARGV; + my $lang = shift @ARGV; $C = ($lang eq '-C'); -# $TXT = ($lang eq '-txt'); } -for $fn (@ARGV) { +our %basenames = (); + +for my $fn (@ARGV) { open(F, "$fn"); - $lastnil = 0; - $lastline = ""; - $incomment = 0; + my $lastnil = 0; + my $lastline = ""; + my $incomment = 0; + my $in_func_head = 0; + my $basename = $fn; + $basename =~ s#.*/##; + if ($basenames{$basename}) { + msg "Duplicate fnames: $fn and $basenames{$basename}.\n"; + } else { + $basenames{$basename} = $fn; + } while (<F>) { ## Warn about windows-style newlines. - # (We insist on lines that end with a single LF character, not - # CR LF.) + # (We insist on lines that end with a single LF character, not + # CR LF.) if (/\r/) { - print " CR:$fn:$.\n"; + msg " CR:$fn:$.\n"; } ## Warn about tabs. - # (We only use spaces) + # (We only use spaces) if (/\t/) { - print " TAB:$fn:$.\n"; + msg " TAB:$fn:$.\n"; } ## Warn about labels that don't have a space in front of them - # (We indent every label at least one space) + # (We indent every label at least one space) if (/^[a-zA-Z_][a-zA-Z_0-9]*:/) { - print "nosplabel:$fn:$.\n"; + msg "nosplabel:$fn:$.\n"; } ## Warn about trailing whitespace. - # (We don't allow whitespace at the end of the line; make your - # editor highlight it for you so you can stop adding it in.) + # (We don't allow whitespace at the end of the line; make your + # editor highlight it for you so you can stop adding it in.) if (/ +$/) { - print "Space\@EOL:$fn:$.\n"; + msg "Space\@EOL:$fn:$.\n"; } ## Warn about control keywords without following space. - # (We put a space after every 'if', 'while', 'for', 'switch', etc) + # (We put a space after every 'if', 'while', 'for', 'switch', etc) if ($C && /\s(?:if|while|for|switch)\(/) { - print " KW(:$fn:$.\n"; + msg " KW(:$fn:$.\n"; } ## Warn about #else #if instead of #elif. # (We only allow #elif) if (($lastline =~ /^\# *else/) and ($_ =~ /^\# *if/)) { - print " #else#if:$fn:$.\n"; + msg " #else#if:$fn:$.\n"; } ## Warn about some K&R violations # (We use K&R-style C, where open braces go on the same line as @@ -53,23 +73,23 @@ for $fn (@ARGV) { # other stuff; # } if (/^\s+\{/ and $lastline =~ /^\s*(if|while|for|else if)/ and - $lastline !~ /\{$/) { - print "non-K&R {:$fn:$.\n"; - } + $lastline !~ /\{$/) { + msg "non-K&R {:$fn:$.\n"; + } if (/^\s*else/ and $lastline =~ /\}$/) { - print " }\\nelse:$fn:$.\n"; - } + msg " }\\nelse:$fn:$.\n"; + } $lastline = $_; ## Warn about unnecessary empty lines. # (Don't put an empty line before a line that contains nothing # but a closing brace.) if ($lastnil && /^\s*}\n/) { - print " UnnecNL:$fn:$.\n"; + msg " UnnecNL:$fn:$.\n"; } ## Warn about multiple empty lines. # (At most one blank line in a row.) if ($lastnil && /^$/) { - print " DoubleNL:$fn:$.\n"; + msg " DoubleNL:$fn:$.\n"; } elsif (/^$/) { $lastnil = 1; } else { @@ -79,7 +99,7 @@ for $fn (@ARGV) { ## accept double-line lines. # (Don't make lines wider than 80 characters, including newline.) if (/^.{80}/) { - print " Wide:$fn:$.\n"; + msg " Wide:$fn:$.\n"; } ### Juju to skip over comments and strings, since the tests ### we're about to do are okay there. @@ -102,48 +122,52 @@ for $fn (@ARGV) { s!"(?:[^\"]+|\\.)*"!"X"!g; next if /^\#/; ## Warn about C++-style comments. - # (Use C style comments only.) + # (Use C style comments only.) if (m!//!) { - # print " //:$fn:$.\n"; + # msg " //:$fn:$.\n"; s!//.*!!; } ## Warn about unquoted braces preceded by non-space. - # (No character except a space should come before a {) + # (No character except a space should come before a {) if (/([^\s'])\{/) { - print " $1\{:$fn:$.\n"; + msg " $1\{:$fn:$.\n"; + } + ## Warn about double semi-colons at the end of a line. + if (/;;$/) { + msg " double semi-colons at the end of $. in $fn\n" } ## Warn about multiple internal spaces. #if (/[^\s,:]\s{2,}[^\s\\=]/) { - # print " X X:$fn:$.\n"; + # msg " X X:$fn:$.\n"; #} ## Warn about { with stuff after. #s/\s+$//; #if (/\{[^\}\\]+$/) { - # print " {X:$fn:$.\n"; + # msg " {X:$fn:$.\n"; #} ## Warn about function calls with space before parens. - # (Don't put a space between the name of a function and its - # arguments.) + # (Don't put a space between the name of a function and its + # arguments.) if (/(\w+)\s\(([A-Z]*)/) { if ($1 ne "if" and $1 ne "while" and $1 ne "for" and $1 ne "switch" and $1 ne "return" and $1 ne "int" and $1 ne "elsif" and $1 ne "WINAPI" and $2 ne "WINAPI" and $1 ne "void" and $1 ne "__attribute__" and $1 ne "op" and - $1 ne "size_t" and $1 ne "double" and + $1 ne "size_t" and $1 ne "double" and $1 ne "uint64_t" and $1 ne "workqueue_reply_t") { - print " fn ():$fn:$.\n"; + msg " fn ():$fn:$.\n"; } } ## Warn about functions not declared at start of line. - # (When you're declaring functions, put "static" and "const" - # and the return type on one line, and the function name at - # the start of a new line.) + # (When you're declaring functions, put "static" and "const" + # and the return type on one line, and the function name at + # the start of a new line.) if ($in_func_head || ($fn !~ /\.h$/ && /^[a-zA-Z0-9_]/ && ! /^(?:const |static )*(?:typedef|struct|union)[^\(]*$/ && ! /= *\{$/ && ! /;$/)) { if (/.\{$/){ - print "fn() {:$fn:$.\n"; + msg "fn() {:$fn:$.\n"; $in_func_head = 0; } elsif (/^\S[^\(]* +\**[a-zA-Z0-9_]+\(/) { $in_func_head = -1; # started with tp fn @@ -151,31 +175,33 @@ for $fn (@ARGV) { $in_func_head = 0; } elsif (/\{/) { if ($in_func_head == -1) { - print "tp fn():$fn:$.\n"; + msg "tp fn():$fn:$.\n"; } $in_func_head = 0; } } - ## Check for forbidden functions except when they are - # explicitly permitted - if (/\bassert\(/ && not /assert OK/) { - print "assert :$fn:$. (use tor_assert)\n"; - } - if (/\bmemcmp\(/ && not /memcmp OK/) { - print "memcmp :$fn:$. (use {tor,fast}_mem{eq,neq,cmp}\n"; - } - # always forbidden. - if (not / OVERRIDE /) { - if (/\bstrcat\(/ or /\bstrcpy\(/ or /\bsprintf\(/) { - print "$& :$fn:$.\n"; - } - if (/\bmalloc\(/ or /\bfree\(/ or /\brealloc\(/ or - /\bstrdup\(/ or /\bstrndup\(/ or /\bcalloc\(/) { - print "$& :$fn:$. (use tor_malloc, tor_free, etc)\n"; - } - } + ## Check for forbidden functions except when they are + # explicitly permitted + if (/\bassert\(/ && not /assert OK/) { + msg "assert :$fn:$. (use tor_assert)\n"; + } + if (/\bmemcmp\(/ && not /memcmp OK/) { + msg "memcmp :$fn:$. (use {tor,fast}_mem{eq,neq,cmp}\n"; + } + # always forbidden. + if (not /\ OVERRIDE\ /) { + if (/\bstrcat\(/ or /\bstrcpy\(/ or /\bsprintf\(/) { + msg "$& :$fn:$.\n"; + } + if (/\bmalloc\(/ or /\bfree\(/ or /\brealloc\(/ or + /\bstrdup\(/ or /\bstrndup\(/ or /\bcalloc\(/) { + msg "$& :$fn:$. (use tor_malloc, tor_free, etc)\n"; + } + } } } close(F); } + +exit $found; diff --git a/scripts/maint/display_callgraph.py b/scripts/maint/display_callgraph.py deleted file mode 100755 index c9001c6d96..0000000000 --- a/scripts/maint/display_callgraph.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/python - -import cPickle - -data = cPickle.load(open("callgraph.pkl")) - -# data = data['modItems'] - -callgraph = data['callgraph'] -closure = data['closure'] -sccs = data['sccs'] -fn_bottle, call_bottle = data['bottlenecks'] - -for n_reachable, fn in sorted(list((len(r), fn) for fn, r in closure.iteritems())): - print "%s can reach %s other functions." %(fn, n_reachable) - - -c = [ (len(component), component) for component in sccs ] -c.sort() - -print "\n================================" - -for n, component in c: - if n < 2: - continue - print "Strongly connected component of size %d:"%n - print component - - -print "\n================================" - -print "====== Number of functions pulled into blob, by function in blob." -fn_bottle.sort() -for n, fn in fn_bottle[-30:]: - print "%3d: %s"%(n, fn) - -print "====== Number of functions pulled into blob, by call in blob." -call_bottle.sort() -for n, fn1, _, fn2 in call_bottle[-30:]: - print "%3d: %s -> %s "%(n, fn2, fn1) - diff --git a/scripts/maint/fallback.blacklist b/scripts/maint/fallback.blacklist deleted file mode 100644 index c9fd8a9236..0000000000 --- a/scripts/maint/fallback.blacklist +++ /dev/null @@ -1,229 +0,0 @@ -# updateFallbackDirs.py directory mirror blacklist -# -# Format: -# [ IPv4[:DirPort] ] [ orport=<ORPort> ] [ id=<ID> ] ... -# [ ipv6=<IPv6>[:<IPv6 ORPort>] ] -# -# If a sufficiently specific group of attributes matches, the directory mirror -# will be excluded: (each group is listed on its own line) -# <IPv4>, <DirPort> -# <IPv4>, <ORPort> -# <ID> -# <IPv6>, <DirPort> -# <IPv6>, <IPv6 ORPort> -# If DirPort and ORPort are not present, the entire IP address is blacklisted. -# (The blacklist overrides the whitelist.) - -# If a relay operator doesn't want their relay to be a FallbackDir, -# enter the following information here: -# <IPv4>:<DirPort> orport=<ORPort> id=<ID> ipv6=<IPv6>:<IPv6 ORPort> - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008364.html -87.181.248.227:9030 orport=443 id=8827944C4BDCBDAC9079803F47823403C11A9B7A - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008368.html -149.18.2.82:9030 orport=9001 id=953DB709F2A2DECC8D7560661F934E64411444F7 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008384.html -80.82.215.199:80 orport=443 id=3BEFAB76461B6B99DCF34C285E933562F5712AE4 ipv6=[2001:4ba0:cafe:a18::1]:443 - -# https://lists.torproject.org/pipermail/tor-relays/2016-January/008515.html -# later opt-out in -# https://lists.torproject.org/pipermail/tor-relays/2016-January/008521.html -5.9.158.75:80 orport=443 id=F1BE15429B3CE696D6807F4D4A58B1BFEC45C822 ipv6=[2a01:4f8:190:514a::2]:443 - -# Email sent directly to teor, verified using relay contact info -5.34.183.168:80 orport=443 id=601C92108A568742A7A6D9473FE3A414F7149070 -217.12.199.208:8080 orport=22 id=BCFB0933367D626715DA32A147F417194A5D48D6 - -# https://lists.torproject.org/pipermail/tor-relays/2016-January/008555.html -62.210.207.124:9030 orport=9001 id=58938B1A5C4029B4415D38A4F36B7724273F4755 ipv6=[2001:bc8:31eb:100::1]:9001 -62.210.207.124:9130 orport=9101 id=338D0AB6DBAB7B529B9C91B2FD770658000693C4 ipv6=[2001:bc8:31eb:100::1]:9101 - -# these fallback candidates fail the consensus download test in a way that -# causes stem to hang (and not respond to ^C, at least on OS X) -# (Is something sending weird responses to DirPort traffic?) -#217.23.14.190:1194 -#151.80.164.147:80 -#148.251.255.92:80 -#78.142.19.59:80 - -# Email sent directly to teor, verified using relay contact info -216.17.99.183:80 orport=443 id=D52CD431CEF28E01B11F545A84347EE45524BCA7 -216.17.99.183:8080 orport=9001 id=EE21F83AB6F76E3B3FFCBA5C2496F789CB84E7C6 -65.19.167.130:80 orport=443 id=890E2EA65455FBF0FAAB4159FAC4412BDCB24295 -65.19.167.131:80 orport=443 id=0DA9BD201766EDB19F57F49F1A013A8A5432C008 -65.19.167.132:80 orport=443 id=12B80ABF019354A9D25EE8BE85EB3C0AD8F7DFC1 -65.19.167.133:80 orport=443 id=C170AE5A886C5A09D6D1CF5CF284653632EEF25D - -# Email sent directly to teor, verified using relay contact info -213.136.83.225:80 orport=443 id=B411027C926A9BFFCF7DA91E3CAF1856A321EFFD -195.154.126.78:80 orport=443 id=F6556156E2B3837248E03FDB770441CF64DBBFBE - -# Email sent directly to teor, verified using relay contact info -178.63.198.113:80 orport=443 id=872B18761953254914F77C71846E8A2623C52591 - -# Email sent directly to teor, verified using relay contact info -63.141.226.34:80 orport=9001 id=5EF131C0C82270F40B756987FDB5D54D9C966238 -185.75.56.103:80 orport=9001 id=3763CE5C3F574670D4296573744F821C0FFFB98E - -# Email sent directly to teor, verified using relay contact info -81.7.14.227:9030 orport=9001 id=BCA197C43A44B7B9D14509637F96A45B13C233D0 - -# Email sent directly to teor, verified using relay contact info -84.245.32.195:9030 orport=9001 id=4CD4DFFEF3971C902A22100D911CAC639BE2EF5C - -# Email sent directly to teor, verified using relay contact info -185.21.217.10:9030 orport=9001 id=41537E1D3DD3CAE86F5A3F0882F1C647FE8FC0A0 - -# Email sent directly to teor, verified using relay contact info -185.21.216.140:9030 orport=9001 id=921DA852C95141F8964B359F774B35502E489869 - -# Email sent directly to teor, verified using relay contact info -62.210.82.44:143 orport=21 id=1C90D3AEADFF3BCD079810632C8B85637924A58E ipv6=[2001:bc8:3d7c::]:21 - -# Email sent directly to teor, verified using relay contact info -46.101.220.161:80 orport=443 id=7DDFE5B2C306B19A79832FBE581EAA245BAE90C6 ipv6=[2a03:b0c0:3:d0::8b:3001]:443 - -# Email sent directly to teor, verified using relay contact info -195.154.107.23:80 orport=443 id=A1F89F26E82209169E4037B035AE7B6C94A49AEB ipv6=[2001:bc8:3829:300::1]:443 -195.154.92.70:80 orport=443 id=E7FF4ECEEFCFE3A40A6D3594898A4A3DE018BBF5 ipv6=[2001:bc8:3829:500::1]:443 -195.154.113.200:80 orport=443 id=D1A4763FA0BD71978901B1951FEE1DC29777F95A ipv6=[2001:bc8:3829:600::1]:443 -195.154.92.155:110 orport=993 id=4477D3466FE136B7FE6F7FF8EBD0D6E2FFE3288B ipv6=[2001:bc8:3829:100::1]:993 -195.154.117.182:110 orport=993 id=B1A0F1143789466AADD5FAE5948C8138548EECEC ipv6=[2001:bc8:3829:400::1]:993 -195.154.97.163:80 orport=443 id=8A2994A63B20813B7724817A8FB8C444D10BA2E2 - -# Email sent directly to teor, verified using relay contact info -5.135.154.206:9030 orport=9001 id=7D67B342DC1158F4CFFEE8BC530A2448848026E3 - -# Email sent directly to teor, verified using relay contact info -85.24.215.117:9030 orport=9001 id=5989521A85C94EE101E88B8DB2E68321673F9405 ipv6=[2001:9b0:20:2106:21a:4aff:fea5:ad05]:9001 - -# Email sent directly to teor, verified using relay contact info -62.210.137.230:8888 orport=8843 id=CD6B850159CFF4C068A8D0F1BA5296AE4EDCAB39 ipv6=[2001:bc8:31d3:100::1]:3443 -62.210.137.230:8080 orport=8443 id=F596E1B1EF98E1DDBBDC934DB722AF54069868F6 ipv6=[2001:bc8:31d3:100::1]:8443 - -# Email sent directly to teor, verified using relay contact info -195.154.99.80:80 orport=443 id=6E7CB6E783C1B67B79D0EBBE7D48BC09BD441201 -195.154.127.60:80 orport=443 id=D74ABE34845190E242EC74BA28B8C89B0A480D4B - -# Email sent directly to teor, verified using relay contact info -212.51.143.20:80 orport=443 id=62DA0256BBC28992D41CBAFB549FFD7C9B846A99 - -# Email sent directly to teor, verified using relay contact info -195.154.90.122:80 orport=443 id=3A0D88024A30152E6F6372CFDF8F9B725F984362 - -# Email sent directly to teor, verified using relay contact info -188.166.118.215:9030 orport=443 id=FB5FF60F5EBA010F8A45AC6ED31A4393718A2C31 ipv6=[2a03:b0c0:2:d0::72:9001]:443 - -# Email sent directly to teor, verified using relay contact info -185.87.185.245:40001 orport=40000 id=2A499AEEA95FB10F07544383D562368E49BE32CA - -# Email sent directly to teor, verified using relay contact info -82.161.109.71:9030 orport=9001 id=BD9CE352648B940E788A8E45393C5400CC3E87E7 - -# Email sent directly to teor, verified using relay contact info -212.83.40.239:9030 orport=9001 id=6DC5616BD3FC463329DCE87DD7AAAEA112C264B5 - -# Email sent directly to teor, verified using relay contact info -178.32.53.53:80 orport=443 id=10582C360E972EE76B0DB1C246F4E892A6BF5465 - -# Email sent directly to teor, verified using relay contact info -85.114.135.20:9030 orport=9001 id=ED8A9291A3139E34BBD35037B082081EC6C26C80 ipv6=[2001:4ba0:fff5:2d::8]:9001 -148.251.128.156:9030 orport=9001 id=E382042E06A0A68AFC533E5AD5FB6867A12DF9FF ipv6=[2a01:4f8:210:238a::8]:9001 -62.210.115.147:9030 orport=9001 id=7F1D94E2C36F8CC595C2AB00022A5AE38171D50B ipv6=[2001:bc8:3182:101::8]:9001 -212.47.250.24:9030 orport=9001 id=33DA0CAB7C27812EFF2E22C9705630A54D101FEB - -# Email sent directly to teor, verified using relay contact info -74.208.220.222:60000 orport=59999 id=4AA22235F0E9B3795A33930343CBB3EDAC60C5B0 - -# Email sent directly to teor, verified using relay contact info -89.163.140.168:9030 orport=9001 id=839C1212DB15723263BE96C83DA7E1B24FA395E8 - -# Email sent directly to teor, verified using relay contact info -212.47.246.211:9030 orport=9001 id=AA34219475E41282095DD3C088009EE562AF14E5 - -# Email sent directly to teor, verified using relay contact info -85.195.235.148:9030 orport=9001 id=103336165A0D2EFCAD3605339843A0A7710B8B92 -85.195.235.148:19030 orport=19001 id=713235638AB6C64715EAFD1B4772685E38AFD52A - -# Email sent directly to teor, verified using relay contact info -163.172.7.30:9030 orport=9001 id=E2EACD4752B2583202F374A34CACC844A3AECAC4 - -# Email sent directly to teor, verified using relay contact info -178.62.90.111:22 orport=25 id=3254D1DC1F1531D9C07C535E4991F38EE99B99E1 - -# Email sent directly to teor, verified using relay contact info -213.200.106.131:9030 orport=4443 id=B07CE79FD215129C381F6645B16E76DCA0845CAB - -# Email sent directly to teor, verified using relay contact info -198.51.75.165:80 orport=9001 id=DABCB84A524A22FDDD3AFCB090E3090CC12D9770 - -# Email sent directly to teor, verified using relay contact info -204.194.29.4:80 orport=9001 id=78C7C299DB4C4BD119A22B87B57D5AF5F3741A79 - -# Email sent directly to teor, verified using relay contact info -104.207.132.109:9030 orport=9001 id=12D5737383C23E756A7AA1A90BB24413BA428DA7 ipv6=[2001:19f0:300:2261::1]:9001 - -# Email sent directly to teor, verified using relay contact info -46.252.25.249:9030 orport=443 id=80DCBB6EF4E86A7CD4FBCBDEE64979645509A610 - -# Email sent directly to teor, verified using relay contact info -176.10.99.200:8080 orport=443 id=2B44FD1742D26E4F28D4CACF1F0CF8A686270E45 -176.10.99.200:8000 orport=22 id=EB79F07792A065D3C534063773E83268E069F5EB -176.10.99.201:667 orport=666 id=3EAAAB35932610411E24FA4317603CB5780B80BC -176.10.99.201:990 orport=989 id=7C3A4CFF09C1981D41173CDE2A2ADD4A5CA109FD -176.10.99.202:992 orport=991 id=615EBC4B48F03858FA50A3E23E5AF569D0D2308A -176.10.99.202:994 orport=993 id=E34E25D958D46DDE5092385B14117C9B301DC0E9 -176.10.99.203:1194 orport=995 id=AD368442E9FF33C08C7407DF2DA7DB958F406CE2 -176.10.99.203:43 orport=53 id=79CF377F0ACEC5F0002D85335E4192B34202A269 -176.10.99.204:1755 orport=1723 id=69DF3CDA1CDA460C17ECAD9D6F0C117A42384FA0 -176.10.99.204:1293 orport=4321 id=3F061400B6FB1F55E7F19BB3C713884D677E55B7 -176.10.99.205:426 orport=425 id=C30B284784BF11D0D58C6A250240EE58D2084AD0 -176.10.99.205:109 orport=110 id=12D17D9F9E30FA901DE68806950A0EA278716CED -176.10.99.206:24 orport=23 id=2C804AAB0C02F971A4386B3A1F2AC00F9E080679 -176.10.99.206:20 orport=21 id=237588726AB6BEA37FF23CA00F5BD178586CA68E -176.10.99.207:3390 orport=3389 id=A838D5B8890B10172429ECE92EB5677DF93DC4DD -176.10.99.207:1415 orport=1414 id=377E5E817A84FAE0F4DC3427805DB2E8A6CBBFC0 -176.10.99.208:390 orport=389 id=7C288587BA0D99CC6B8537CDC2C4639FA827B907 -176.10.99.208:3307 orport=3306 id=1F0D2A44C56F42816DED2022EFD631878C29905B -176.10.99.209:1434 orport=1433 id=BDA7A91FF3806DE5109FDAE74CFEFB3BABB9E10F -176.10.99.209:220 orport=219 id=B8C2030001D832066A648269CFBA94171951D34B - -# Email sent directly to teor, verified using relay contact info -78.193.40.205:8080 orport=8443 id=C91450840E75AC1B654A3096744338A573A239C6 - -# Email sent directly to teor, verified using relay contact info -37.187.22.172:9030 orport=9035 id=335E4117BD9A4966403C2AFA31CFDD1BC13BD46A - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008367.html -# Email sent directly to teor to opt-out -88.198.38.226:22 orport=443 id=4B9E2C56FB42B891794FE2CD2FCAD08A320CC3BB ipv6=[2a01:4f8:a0:1351::2]:80 -213.239.210.204:22 orport=443 id=5BFDECCE9B4A23AE14EC767C5A2C1E10558B00B9 ipv6=[2a01:4f8:a0:9474::2]:80 -213.239.220.25:22 orport=443 id=BEE2317AE127EB681C5AE1551C1EA0630580638A ipv6=[2a01:4f8:a0:710c::2]:80 -85.10.201.38:22 orport=443 id=F6279A203C1950ACF592322A235647A05BFBCF91 ipv6=[2a01:4f8:a0:43cc::2]:80 - -# Email sent directly to teor, verified using relay contact info -88.190.208.4:30555 orport=30556 id=030A6EB24725C05D8E0FCE21923CBA5223E75E0E - -# Fallback was on 0.2.8.2-alpha list, but changed fingerprint before 0.2.8.5 -46.101.102.71:80 orport=443 id=9504CB22EEB25D344DE63CB7A6F2C46F895C3686 ipv6=[2a03:b0c0:3:d0::2ed:7001]:9050 -# Also blacklist anything with the new fingerprint -id=9C8A123081EFBE022EF795630F447839DDFDDDEC - -# Fallbacks were on 0.2.8.2-alpha list, but downloads were slow before 0.2.8.5 -185.96.88.29:80 orport=443 id=86C281AD135058238D7A337D546C902BE8505DDE -178.62.36.64:9030 orport=9001 id=B87C84E38DAECFFFFDE98E5AEE5786AFDC748F2C - -# Fallback was on 0.2.8.2-alpha list, but changed address before 0.2.8.5 -84.219.173.60:9030 orport=443 id=855BC2DABE24C861CD887DB9B2E950424B49FC34 -# Also blacklist anything with the new address -84.216.235.55:9030 orport=443 - -# Fallbacks were on 0.2.8.2-alpha list, but disappeared before 0.2.8.5 -81.7.17.171:80 orport=443 id=CFECDDCA990E3EF7B7EC958B22441386B6B8D820 ipv6=[2a02:180:1:1::517:11ab]:443 -51.254.215.121:80 orport=443 id=262B66AD25C79588AD1FC8ED0E966395B47E5C1D -185.100.85.138:80 orport=46356 id=5C4DF16A0029CC4F67D3E127356E68F219269859 - -# Fallback was on 0.2.8.2-alpha list, but opted-out before 0.2.8.6 -37.187.1.149:9030 orport=9001 id=08DC0F3C6E3D9C527C1FC8745D35DD1B0DE1875D ipv6=[2001:41d0:a:195::1]:9001 diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist deleted file mode 100644 index c801e46b15..0000000000 --- a/scripts/maint/fallback.whitelist +++ /dev/null @@ -1,770 +0,0 @@ -# updateFallbackDirs.py directory mirror whitelist -# -# Format: -# IPv4:DirPort orport=<ORPort> id=<ID> [ ipv6=<IPv6>:<IPv6 ORPort> ] -# -# All attributes must match for the directory mirror to be included. -# If the fallback has an ipv6 key, the whitelist line must also have -# it, and vice versa, otherwise they don't match. -# (The blacklist overrides the whitelist.) - -# To replace this list with the hard-coded fallback list (for testing), use -# a command similar to: -# cat src/or/fallback_dirs.inc | grep \" | grep -v weight | tr -d '\n' | \ -# sed 's/"" / /g' | sed 's/""/"/g' | tr \" '\n' | grep -v '^$' \ -# > scripts/maint/fallback.whitelist -# -# When testing before a release, exclusions due to changed details will result -# in a warning, unless the IPv4 address or port change happened recently. -# Then it is only logged at info level, as part of the eligibility check. -# Exclusions due to stability also are only shown at info level. -# -# Add the number of selected, slow, and excluded relays, and compare that to -# the number of hard-coded relays. If it's less, use info-level logs to find -# out why each of the missing relays was excluded. - -# If a relay operator wants their relay to be a FallbackDir, -# enter the following information here: -# <IPv4>:<DirPort> orport=<ORPort> id=<ID> [ ipv6=<IPv6>:<IPv6 ORPort> ] - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008362.html -78.47.18.110:443 orport=80 id=F8D27B163B9247B232A2EEE68DD8B698695C28DE -131.188.40.188:443 orport=80 id=EBE718E1A49EE229071702964F8DB1F318075FF8 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008366.html -5.39.88.19:9030 orport=9001 id=7CB8C31432A796731EA7B6BF4025548DFEB25E0C ipv6=[2001:41d0:8:9a13::1]:9050 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008370.html -# https://lists.torproject.org/pipermail/tor-relays/2016-January/008517.html -# https://lists.torproject.org/pipermail/tor-relays/2016-January/008555.html -62.210.124.124:9030 orport=9001 id=86E78DD3720C78DA8673182EF96C54B162CD660C ipv6=[2001:bc8:3f23:100::1]:9001 -62.210.124.124:9130 orport=9101 id=2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E ipv6=[2001:bc8:3f23:100::1]:9101 -212.47.237.95:9030 orport=9001 id=3F5D8A879C58961BB45A3D26AC41B543B40236D6 -212.47.237.95:9130 orport=9101 id=6FB38EB22E57EF7ED5EF00238F6A48E553735D88 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008372.html -# IPv6 tunnel available on request (is this a good idea?) -108.53.208.157:80 orport=443 id=4F0DB7E687FC7C0AE55C8F243DA8B0EB27FBF1F2 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008373.html -167.114.35.28:9030 orport=9001 id=E65D300F11E1DB12C534B0146BDAB6972F1A8A48 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008374.html -170.130.1.7:9030 orport=9001 id=FA3415659444AE006E7E9E5375E82F29700CFDFD - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008378.html -144.76.14.145:110 orport=143 id=14419131033443AE6E21DA82B0D307F7CAE42BDB ipv6=[2a01:4f8:190:9490::dead]:443 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008379.html -# Email sent directly to teor, verified using relay contact info -91.121.84.137:4951 orport=4051 id=6DE61A6F72C1E5418A66BFED80DFB63E4C77668F - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008380.html -5.175.233.86:80 orport=443 id=5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008381.html -# Sent additional email to teor with more relays -178.254.44.135:9030 orport=9001 id=8FA37B93397015B2BC5A525C908485260BE9F422 -178.254.20.134:80 orport=443 id=9F5068310818ED7C70B0BC4087AB55CB12CB4377 -178.254.20.134:9030 orport=9001 id=2CE96A8A1DA032664C90F574AFFBECE18A6E8DFC -178.254.44.135:80 orport=443 id=AE6A8C18E7499B586CD36246AC4BCAFFBBF93AB2 -178.254.13.126:80 orport=443 id=F9246DEF2B653807236DA134F2AEAB103D58ABFE -178.254.13.126:9030 orport=9001 id=0C475BA4D3AA3C289B716F95954CAD616E50C4E5 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008382.html -51.255.33.237:9091 orport=9001 id=A360C21FA87FFA2046D92C17086A6B47E5C68109 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008383.html -81.7.14.246:80 orport=443 id=CE75BF0972ADD52AF8807602374E495C815DB304 ipv6=[2a02:180:a:51::dead]:443 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008384.html -# Sent additional email to teor with fingerprint change -149.202.98.161:80 orport=443 id=FC64CD763F8C1A319BFBBF62551684F4E1E42332 ipv6=[2001:41d0:8:4528::161]:443 -193.111.136.162:80 orport=443 id=C79552275DFCD486B942510EF663ED36ACA1A84B ipv6=[2001:4ba0:cafe:10d0::1]:443 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008416.html -185.100.84.212:80 orport=443 id=330CD3DB6AD266DC70CDB512B036957D03D9BC59 ipv6=[2a06:1700:0:7::1]:443 - -# https://lists.torproject.org/pipermail/tor-relays/2015-December/008417.html -178.16.208.56:80 orport=443 id=2CDCFED0142B28B002E89D305CBA2E26063FADE2 ipv6=[2a00:1c20:4089:1234:cd49:b58a:9ebe:67ec]:443 -178.16.208.57:80 orport=443 id=92CFD9565B24646CAC2D172D3DB503D69E777B8A ipv6=[2a00:1c20:4089:1234:7825:2c5d:1ecd:c66f]:443 - -# https://lists.torproject.org/pipermail/tor-relays/2016-January/008513.html -178.62.173.203:9030 orport=9001 id=DD85503F2D1F52EF9EAD621E942298F46CD2FC10 ipv6=[2a03:b0c0:0:1010::a4:b001]:9001 - -# https://lists.torproject.org/pipermail/tor-relays/2016-January/008534.html -5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB ipv6=[2a01:4f8:162:51e2::2]:9001 - -# https://lists.torproject.org/pipermail/tor-relays/2016-January/008542.html -178.62.199.226:80 orport=443 id=CBEFF7BA4A4062045133C053F2D70524D8BBE5BE ipv6=[2a03:b0c0:2:d0::b7:5001]:443 - -# Emails sent directly to teor, verified using relay contact info -217.12.199.208:80 orport=443 id=DF3AED4322B1824BF5539AE54B2D1B38E080FF05 - -# Email sent directly to teor, verified using relay contact info -94.23.204.175:9030 orport=9001 id=5665A3904C89E22E971305EE8C1997BCA4123C69 - -# https://twitter.com/binarytenshi/status/717952514327453697 -94.126.23.174:9030 orport=9001 id=6FC6F08270D565BE89B7C819DD8E2D487397C073 - -# Email sent directly to teor, verified using relay contact info -171.25.193.78:80 orport=443 id=A478E421F83194C114F41E94F95999672AED51FE ipv6=[2001:67c:289c:3::78]:443 -171.25.193.77:80 orport=443 id=A10C4F666D27364036B562823E5830BC448E046A ipv6=[2001:67c:289c:3::77]:443 -171.25.193.131:80 orport=443 id=79861CF8522FC637EF046F7688F5289E49D94576 -171.25.193.20:80 orport=443 id=DD8BD7307017407FCC36F8D04A688F74A0774C02 ipv6=[2001:67c:289c::20]:443 -# OK, but same machine as 79861CF8522FC637EF046F7688F5289E49D94576 -#171.25.193.132:80 orport=443 id=01C67E0CA8F97111E652C7564CB3204361FFFAB8 -# OK, but same machine as DD8BD7307017407FCC36F8D04A688F74A0774C02 -#171.25.193.25:80 orport=443 id=185663B7C12777F052B2C2D23D7A239D8DA88A0F ipv6=[2001:67c:289c::25]:443 - -# Email sent directly to teor, verified using relay contact info -212.47.229.2:9030 orport=9001 id=20462CBA5DA4C2D963567D17D0B7249718114A68 -93.115.97.242:9030 orport=9001 id=B5212DB685A2A0FCFBAE425738E478D12361710D -46.28.109.231:9030 orport=9001 id=F70B7C5CD72D74C7F9F2DC84FA9D20D51BA13610 ipv6=[2a02:2b88:2:1::4205:42]:9001 - -# Email sent directly to teor, verified using relay contact info -85.235.250.88:80 orport=443 id=72B2B12A3F60408BDBC98C6DF53988D3A0B3F0EE -185.96.180.29:80 orport=443 id=F93D8F37E35C390BCAD9F9069E13085B745EC216 - -# Email sent directly to teor, verified using relay contact info -185.11.180.67:80 orport=9001 id=794D8EA8343A4E820320265D05D4FA83AB6D1778 - -# Email sent directly to teor, verified using relay contact info -178.16.208.62:80 orport=443 id=5CF8AFA5E4B0BB88942A44A3F3AAE08C3BDFD60B ipv6=[2a00:1c20:4089:1234:a6a4:2926:d0af:dfee]:443 -46.165.221.166:80 orport=443 id=EE5F897C752D46BCFF531641B853FC6BC78DD4A7 -178.16.208.60:80 orport=443 id=B44FBE5366AD98B46D829754FA4AC599BAE41A6A ipv6=[2a00:1c20:4089:1234:67bc:79f3:61c0:6e49]:443 -178.16.208.55:80 orport=443 id=C4AEA05CF380BAD2230F193E083B8869B4A29937 ipv6=[2a00:1c20:4089:1234:7b2c:11c5:5221:903e]:443 -178.16.208.61:80 orport=443 id=3B52392E2256C35CDCF7801FF898FC88CE6D431A ipv6=[2a00:1c20:4089:1234:2712:a3d0:666b:88a6]:443 -81.89.96.88:80 orport=443 id=55ED4BB49F6D3F36D8D9499BE43500E017A5EF82 ipv6=[2a02:180:1:1:14c5:b0b7:2d7d:5f3a]:443 -209.222.8.196:80 orport=443 id=C86D2F3DEFE287A0EEB28D4887AF14E35C172733 ipv6=[2001:19f0:1620:41c1:426c:5adf:2ed5:4e88]:443 -81.89.96.89:80 orport=443 id=28651F419F5A1CF74511BB500C58112192DD4943 ipv6=[2a02:180:1:1:2ced:24e:32ea:a03b]:443 -46.165.221.166:9030 orport=9001 id=8C7106C880FE8AA1319DD71B59623FCB8914C9F1 -178.16.208.62:80 orport=443 id=5CF8AFA5E4B0BB88942A44A3F3AAE08C3BDFD60B ipv6=[2a00:1c20:4089:1234:a6a4:2926:d0af:dfee]:443" -46.165.221.166:80 orport=443 id=EE5F897C752D46BCFF531641B853FC6BC78DD4A7 -178.16.208.60:80 orport=443 id=B44FBE5366AD98B46D829754FA4AC599BAE41A6A ipv6=[2a00:1c20:4089:1234:67bc:79f3:61c0:6e49]:443 -178.16.208.55:80 orport=443 id=C4AEA05CF380BAD2230F193E083B8869B4A29937 ipv6=[2a00:1c20:4089:1234:7b2c:11c5:5221:903e]:443 -178.16.208.61:80 orport=443 id=3B52392E2256C35CDCF7801FF898FC88CE6D431A ipv6=[2a00:1c20:4089:1234:2712:a3d0:666b:88a6]:443 -81.89.96.88:80 orport=443 id=55ED4BB49F6D3F36D8D9499BE43500E017A5EF82 ipv6=[2a02:180:1:1:14c5:b0b7:2d7d:5f3a]:443 -209.222.8.196:80 orport=443 id=C86D2F3DEFE287A0EEB28D4887AF14E35C172733 ipv6=[2001:19f0:1620:41c1:426c:5adf:2ed5:4e88]:443 -81.89.96.89:80 orport=443 id=28651F419F5A1CF74511BB500C58112192DD4943 ipv6=[2a02:180:1:1:2ced:24e:32ea:a03b]:443 -46.165.221.166:9030 orport=9001 id=8C7106C880FE8AA1319DD71B59623FCB8914C9F1 -178.16.208.56:80 orport=443 id=2CDCFED0142B28B002E89D305CBA2E26063FADE2 ipv6=[2a00:1c20:4089:1234:cd49:b58a:9ebe:67ec]:443 -178.16.208.58:80 orport=443 id=A4C98CEA3F34E05299417E9F885A642C88EF6029 ipv6=[2a00:1c20:4089:1234:cdae:1b3e:cc38:3d45]:443 -178.16.208.57:80 orport=443 id=92CFD9565B24646CAC2D172D3DB503D69E777B8A ipv6=[2a00:1c20:4089:1234:7825:2c5d:1ecd:c66f]:443 -178.16.208.59:80 orport=443 id=136F9299A5009A4E0E96494E723BDB556FB0A26B ipv6=[2a00:1c20:4089:1234:bff6:e1bb:1ce3:8dc6]:443 - -# Email sent directly to teor, verified using relay contact info -195.154.8.111:80 orport=443 id=FCB6695F8F2DC240E974510A4B3A0F2B12AB5B64 -51.255.235.246:80 orport=443 id=9B99C72B02AF8E3E5BE3596964F9CACD0090D132 -5.39.76.158:80 orport=443 id=C41F60F8B00E7FEF5CCC5BC6BB514CA1B8AAB651 - -# Email sent directly to teor, verified using relay contact info -109.163.234.5:80 orport=443 id=5C84C35936B7100B949AC75764EEF1352550550B -109.163.234.7:80 orport=443 id=C46524E586E1B997329703D356C07EE12B28C722 -109.163.234.9:80 orport=443 id=5714542DCBEE1DD9864824723638FD44B2122CEA -77.247.181.162:80 orport=443 id=7BB160A8F54BD74F3DA5F2CE701E8772B841859D -109.163.234.4:80 orport=443 id=6B1E001929AF4DDBB747D02EC28340792B7724A6 -77.247.181.164:80 orport=443 id=10E13E340651D0EF66B4DEBF610B3C0981168107 -109.163.234.8:80 orport=443 id=20B0038D7A2FD73C696922551B8344CB0893D1F8 -77.247.181.166:80 orport=443 id=06E123865C590189B3181114F23F0F13A7BC0E69 -109.163.234.2:80 orport=443 id=B4F883DB3D478C7AE569C9F6CB766FD58650DC6A -62.102.148.67:80 orport=443 id=4A0C3E177AF684581EF780981AEAF51A98A6B5CF -109.163.234.5:80 orport=443 id=5C84C35936B7100B949AC75764EEF1352550550B -109.163.234.7:80 orport=443 id=C46524E586E1B997329703D356C07EE12B28C722 -109.163.234.9:80 orport=443 id=5714542DCBEE1DD9864824723638FD44B2122CEA -77.247.181.162:80 orport=443 id=7BB160A8F54BD74F3DA5F2CE701E8772B841859D -109.163.234.4:80 orport=443 id=6B1E001929AF4DDBB747D02EC28340792B7724A6 -77.247.181.164:80 orport=443 id=10E13E340651D0EF66B4DEBF610B3C0981168107 -109.163.234.8:80 orport=443 id=20B0038D7A2FD73C696922551B8344CB0893D1F8 -77.247.181.166:80 orport=443 id=06E123865C590189B3181114F23F0F13A7BC0E69 -109.163.234.2:80 orport=443 id=B4F883DB3D478C7AE569C9F6CB766FD58650DC6A -62.102.148.67:80 orport=443 id=4A0C3E177AF684581EF780981AEAF51A98A6B5CF - -# https://twitter.com/biotimylated/status/718994247500718080 -212.47.252.149:9030 orport=9001 id=2CAC39BAA996791CEFAADC9D4754D65AF5EB77C0 - -# Email sent directly to teor, verified using relay contact info -46.165.230.5:80 orport=443 id=A0F06C2FADF88D3A39AA3072B406F09D7095AC9E - -# Email sent directly to teor, verified using relay contact info -94.242.246.24:23 orport=8080 id=EC116BCB80565A408CE67F8EC3FE3B0B02C3A065 ipv6=[2a01:608:ffff:ff07::1:24]:9004 -176.126.252.11:443 orport=9001 id=B0279A521375F3CB2AE210BDBFC645FDD2E1973A ipv6=[2a02:59e0:0:7::11]:9003 -176.126.252.12:21 orport=8080 id=379FB450010D17078B3766C2273303C358C3A442 ipv6=[2a02:59e0:0:7::12]:81 -94.242.246.23:443 orport=9001 id=F65E0196C94DFFF48AFBF2F5F9E3E19AAE583FD0 ipv6=[2a01:608:ffff:ff07::1:23]:9003 -85.248.227.164:444 orport=9002 id=B84F248233FEA90CAD439F292556A3139F6E1B82 ipv6=[2a00:1298:8011:212::164]:9004 -85.248.227.163:443 orport=9001 id=C793AB88565DDD3C9E4C6F15CCB9D8C7EF964CE9 ipv6=[2a00:1298:8011:212::163]:9003 - -# Email sent directly to teor, verified using relay contact info -148.251.190.229:9030 orport=9010 id=BF0FB582E37F738CD33C3651125F2772705BB8E8 ipv6=[2a01:4f8:211:c68::2]:9010 - -# Email sent directly to teor, verified using relay contact info -5.79.68.161:81 orport=443 id=9030DCF419F6E2FBF84F63CBACBA0097B06F557E ipv6=[2001:1af8:4700:a012:1::1]:443 -5.79.68.161:9030 orport=9001 id=B7EC0C02D7D9F1E31B0C251A6B058880778A0CD1 ipv6=[2001:1af8:4700:a012:1::1]:9001 - -# Email sent directly to teor, verified using relay contact info -62.210.92.11:9030 orport=9001 id=0266B0660F3F20A7D1F3D8335931C95EF50F6C6B ipv6=[2001:bc8:338c::1]:9001 -62.210.92.11:9130 orport=9101 id=387B065A38E4DAA16D9D41C2964ECBC4B31D30FF ipv6=[2001:bc8:338c::1]:9101 - -# Email sent directly to teor, verified using relay contact info -188.165.194.195:9030 orport=9001 id=49E7AD01BB96F6FE3AB8C3B15BD2470B150354DF - -# Message sent directly to teor, verified using relay contact info -95.215.44.110:80 orport=443 id=D56AA4A1AA71961F5279FB70A6DCF7AD7B993EB5 -95.215.44.122:80 orport=443 id=998D8FE06B867AA3F8D257A7D28FFF16964D53E2 -95.215.44.111:80 orport=443 id=A7C7FD510B20BC8BE8F2A1D911364E1A23FBD09F - -# Email sent directly to teor, verified using relay contact info -86.59.119.88:80 orport=443 id=ACD889D86E02EDDAB1AFD81F598C0936238DC6D0 - -# Email sent directly to teor, verified using relay contact info -144.76.73.140:9030 orport=9001 id=6A640018EABF3DA9BAD9321AA37C2C87BBE1F907 - -# Email sent directly to teor, verified using relay contact info -193.11.164.243:9030 orport=9001 id=FFA72BD683BC2FCF988356E6BEC1E490F313FB07 ipv6=[2001:6b0:7:125::243]:9001 -109.105.109.162:52860 orport=60784 id=32EE911D968BE3E016ECA572BB1ED0A9EE43FC2F ipv6=[2001:948:7:2::163]:5001 - -# Email sent directly to teor, verified using relay contact info -146.0.32.144:9030 orport=9001 id=35E8B344F661F4F2E68B17648F35798B44672D7E - -# Email sent directly to teor, verified using relay contact info -46.252.26.2:45212 orport=49991 id=E589316576A399C511A9781A73DA4545640B479D - -# Email sent directly to teor, verified using relay contact info -89.187.142.208:80 orport=443 id=64186650FFE4469EBBE52B644AE543864D32F43C - -# Email sent directly to teor, verified using relay contact info -212.51.134.123:9030 orport=9001 id=50586E25BE067FD1F739998550EDDCB1A14CA5B2 ipv6=[2a02:168:6e00:0:3a60:77ff:fe9c:8bd1]:9001 - -# Email sent directly to teor, verified using relay contact info -46.101.143.173:80 orport=443 id=F960DF50F0FD4075AC9B505C1D4FFC8384C490FB - -# Email sent directly to teor, verified using relay contact info -217.79.190.25:9030 orport=9090 id=361D33C96D0F161275EE67E2C91EE10B276E778B - -# Email sent directly to teor, verified using relay contact info -193.171.202.146:9030 orport=9001 id=01A9258A46E97FF8B2CAC7910577862C14F2C524 - -# Email sent directly to teor, verified using relay contact info -197.231.221.211:9030 orport=9001 id=BC630CBBB518BE7E9F4E09712AB0269E9DC7D626 - -# Email sent directly to teor, verified using relay contact info -185.61.138.18:8080 orport=4443 id=2541759BEC04D37811C2209A88E863320271EC9C - -# Email sent directly to teor, verified using relay contact info -193.11.114.45:9031 orport=9002 id=80AAF8D5956A43C197104CEF2550CD42D165C6FB -193.11.114.43:9030 orport=9001 id=12AD30E5D25AA67F519780E2111E611A455FDC89 ipv6=[2001:6b0:30:1000::99]:9050 -193.11.114.46:9032 orport=9003 id=B83DC1558F0D34353BB992EF93AFEAFDB226A73E - -# Email sent directly to teor, verified using relay contact info -144.76.26.175:9012 orport=9011 id=2BA2C8E96B2590E1072AECE2BDB5C48921BF8510 - -# Email sent directly to teor, verified using relay contact info -37.221.162.226:9030 orport=9001 id=D64366987CB39F61AD21DBCF8142FA0577B92811 - -# Email sent directly to teor, verified using relay contact info -91.219.237.244:80 orport=443 id=92ECC9E0E2AF81BB954719B189AC362E254AD4A5 - -# Email sent directly to teor, verified using relay contact info -185.21.100.50:9030 orport=9001 id=58ED9C9C35E433EE58764D62892B4FFD518A3CD0 ipv6=[2a00:1158:2:cd00:0:74:6f:72]:443 - -# Email sent directly to teor, verified using relay contact info -193.35.52.53:9030 orport=9001 id=DAA39FC00B196B353C2A271459C305C429AF09E4 - -# Email sent directly to teor, verified using relay contact info -134.119.3.164:9030 orport=9001 id=D1B8AAA98C65F3DF7D8BB3AF881CAEB84A33D8EE - -# Email sent directly to teor, verified using relay contact info -81.7.10.93:31336 orport=31337 id=99E246DB480B313A3012BC3363093CC26CD209C7 - -# Email sent directly to teor, verified using relay contact info -178.62.22.36:80 orport=443 id=A0766C0D3A667A3232C7D569DE94A28F9922FCB1 ipv6=[2a03:b0c0:1:d0::174:1]:9050 -188.166.23.127:80 orport=443 id=3771A8154DEA98D551607806C80A209CDAA74535 ipv6=[2a03:b0c0:2:d0::27b:7001]:9050 -198.199.64.217:80 orport=443 id=FAD306BAA59F6A02783F8606BDAA431F5FF7D1EA ipv6=[2604:a880:400:d0::1a9:b001]:9050 -159.203.32.149:80 orport=443 id=55C7554AFCEC1062DCBAC93E67B2E03C6F330EFC ipv6=[2604:a880:cad:d0::105:f001]:9050 - -# Email sent directly to teor, verified using relay contact info -5.196.31.80:9030 orport=9900 id=DFB2EB472643FAFCD5E73D2E37D51DB67203A695 ipv6=[2001:41d0:52:400::a65]:9900 - -# Email sent directly to teor, verified using relay contact info -188.138.112.60:1433 orport=1521 id=C414F28FD2BEC1553024299B31D4E726BEB8E788 - -# Email sent directly to teor, verified using relay contact info -213.61.66.118:9031 orport=9001 id=30648BC64CEDB3020F4A405E4AB2A6347FB8FA22 -213.61.66.117:9032 orport=9002 id=6E44A52E3D1FF7683FE5C399C3FB5E912DE1C6B4 -213.61.66.115:9034 orport=9004 id=480CCC94CEA04D2DEABC0D7373868E245D4C2AE2 -213.61.66.116:9033 orport=9003 id=A9DEB920B42B4EC1DE6249034039B06D61F38690 - -# Email sent directly to teor, verified using relay contact info -136.243.187.165:9030 orport=443 id=1AC65257D7BFDE7341046625470809693A8ED83E - -# Email sent directly to teor, verified using relay contact info -212.47.230.49:9030 orport=9001 id=3D6D0771E54056AEFC28BB1DE816951F11826E97 - -# Email sent directly to teor, verified using relay contact info -176.31.180.157:143 orport=22 id=E781F4EC69671B3F1864AE2753E0890351506329 ipv6=[2001:41d0:8:eb9d::1]:22 - -# Email sent directly to teor, verified using relay contact info -192.99.55.69:80 orport=443 id=0682DE15222A4A4A0D67DBA72A8132161992C023 -192.99.59.140:80 orport=443 id=3C9148DA49F20654730FAC83FFF693A4D49D0244 -51.254.215.13:80 orport=443 id=73C30C8ABDD6D9346C822966DE73B9F82CB6178A -51.254.215.129:80 orport=443 id=7B4491D05144B20AE8519AE784B94F0525A8BB79 -192.99.59.139:80 orport=443 id=82EC878ADA7C205146B9F5193A7310867FAA0D7B -51.254.215.124:80 orport=443 id=98999EBE89B5FA9AA0C58421F0B46C3D0AF51CBA -51.254.214.208:80 orport=443 id=C3F0D1417848EAFC41277A73DEB4A9F2AEC23DDF -192.99.59.141:80 orport=443 id=F45426551795B9DA78BEDB05CD5F2EACED8132E4 -192.99.59.14:80 orport=443 id=161A1B29A37EBF096D2F8A9B1E176D6487FE42AE - -# Email sent directly to teor, verified using relay contact info -151.80.42.103:9030 orport=9001 id=9007C1D8E4F03D506A4A011B907A9E8D04E3C605 ipv6=[2001:41d0:e:f67::114]:9001 - -# Email sent directly to teor, verified using relay contact info -5.39.92.199:80 orport=443 id=0BEA4A88D069753218EAAAD6D22EA87B9A1319D6 - -# Email sent directly to teor, verified using relay contact info -176.31.159.231:80 orport=443 id=D5DBCC0B4F029F80C7B8D33F20CF7D97F0423BB1 -176.31.159.230:80 orport=443 id=631748AFB41104D77ADBB7E5CD4F8E8AE876E683 -195.154.79.128:80 orport=443 id=C697612CA5AED06B8D829FCC6065B9287212CB2F -195.154.9.161:80 orport=443 id=B6295A9960F89BD0C743EEBC5670450EA6A34685 -46.148.18.74:8080 orport=443 id=6CACF0B5F03C779672F3C5C295F37C8D234CA3F7 - -# Email sent directly to teor, verified using relay contact info -37.187.102.108:9090 orport=5550 id=F4263275CF54A6836EE7BD527B1328836A6F06E1 -212.47.241.21:80 orport=443 id=892F941915F6A0C6E0958E52E0A9685C190CF45C - -# Email sent directly to teor, verified using relay contact info -195.191.233.221:80 orport=443 id=DE134FC8E5CC4EC8A5DE66934E70AC9D70267197 - -# Email sent directly to teor, verified using relay contact info -62.210.238.33:9030 orport=9001 id=FDF845FC159C0020E2BDDA120C30C5C5038F74B4 - -# Email sent directly to teor, verified using relay contact info -37.157.195.87:8030 orport=443 id=12FD624EE73CEF37137C90D38B2406A66F68FAA2 - -# Email sent directly to teor, verified using relay contact info -37.187.7.74:80 orport=443 id=AEA43CB1E47BE5F8051711B2BF01683DB1568E05 ipv6=[2001:41d0:a:74a::1]:443 - -# Email sent directly to teor, verified using relay contact info -185.66.250.141:9030 orport=9001 id=B1726B94885CE3AC3910CA8B60622B97B98E2529 - -# Email sent directly to teor, verified using relay contact info -185.104.120.7:9030 orport=443 id=445F1C853966624FB3CF1E12442570DC553CC2EC ipv6=[2a06:3000::120:7]:443 -185.104.120.2:9030 orport=21 id=518FF8708698E1DA09C823C36D35DF89A2CAD956 -185.104.120.4:9030 orport=9001 id=F92B3CB9BBE0CB22409843FB1AE4DBCD5EFAC835 -185.104.120.3:9030 orport=21 id=707C1B61AC72227B34487B56D04BAA3BA1179CE8 ipv6=[2a06:3000::120:3]:21 - -# Email sent directly to teor, verified using relay contact info -37.187.102.186:9030 orport=9001 id=489D94333DF66D57FFE34D9D59CC2D97E2CB0053 ipv6=[2001:41d0:a:26ba::1]:9001 - -# Email sent directly to teor, verified using relay contact info -5.35.251.247:9030 orport=9001 id=9B1F5187DFBA89DC24B37EA7BF896C12B43A27AE - -# Email sent directly to teor, verified using relay contact info -198.96.155.3:8080 orport=5001 id=BCEDF6C193AA687AE471B8A22EBF6BC57C2D285E - -# Email sent directly to teor, verified using relay contact info -212.83.154.33:8888 orport=443 id=3C79699D4FBC37DE1A212D5033B56DAE079AC0EF -212.83.154.33:8080 orport=8443 id=322C6E3A973BC10FC36DE3037AD27BC89F14723B - -# Email sent directly to teor, verified using relay contact info -51.255.41.65:9030 orport=9001 id=9231DF741915AA1630031A93026D88726877E93A - -# Email sent directly to teor, verified using relay contact info -78.142.142.246:80 orport=443 id=5A5E03355C1908EBF424CAF1F3ED70782C0D2F74 - -# Email sent directly to teor, verified using relay contact info -195.154.97.91:80 orport=443 id=BD33C50D50DCA2A46AAED54CA319A1EFEBF5D714 - -# Email sent directly to teor, verified using relay contact info -62.210.129.246:80 orport=443 id=79E169B25E4C7CE99584F6ED06F379478F23E2B8 - -# Email sent directly to teor, verified using relay contact info -5.196.74.215:9030 orport=9001 id=5818055DFBAF0FA7F67E8125FD63E3E7F88E28F6 - -# Email sent directly to teor, verified using relay contact info -212.47.233.86:9030 orport=9001 id=B4CAFD9CBFB34EC5DAAC146920DC7DFAFE91EA20 - -# Email sent directly to teor, verified using relay contact info -85.214.206.219:9030 orport=9001 id=98F8D5F359949E41DE8DF3DBB1975A86E96A84A0 - -# Email sent directly to teor, verified using relay contact info -46.166.170.4:80 orport=443 id=19F42DB047B72C7507F939F5AEA5CD1FA4656205 -46.166.170.5:80 orport=443 id=DA705AD4591E7B4708FA2CAC3D53E81962F3E6F6 - -# Email sent directly to teor, verified using relay contact info -5.189.157.56:80 orport=443 id=77F6D6A6B6EAFB8F5DADDC07A918BBF378ED6725 - -# Email sent directly to teor, verified using relay contact info -46.28.110.244:80 orport=443 id=9F7D6E6420183C2B76D3CE99624EBC98A21A967E -185.13.39.197:80 orport=443 id=001524DD403D729F08F7E5D77813EF12756CFA8D -95.130.12.119:80 orport=443 id=587E0A9552E4274B251F29B5B2673D38442EE4BF - -# Email sent directly to teor, verified using relay contact info -212.129.62.232:80 orport=443 id=B143D439B72D239A419F8DCE07B8A8EB1B486FA7 - -# Email sent directly to teor, verified using relay contact info -91.219.237.229:80 orport=443 id=1ECD73B936CB6E6B3CD647CC204F108D9DF2C9F7 - -# Email sent directly to teor, verified using relay contact info -# Suitable, check with operator before adding -#212.47.240.10:82 orport=443 id=2A4C448784F5A83AFE6C78DA357D5E31F7989DEB -212.47.240.10:81 orport=993 id=72527E3242CB15AADE28374AE0D35833FC083F60 -163.172.131.88:80 orport=443 id=AD253B49E303C6AB1E048B014392AC569E8A7DAE ipv6=[2001:bc8:4400:2100::2:1009]:443 -# Suitable, check with operator before adding -#163.172.131.88:81 orport=993 id=D5F3FB17504744FB7ECEF46F4B1D155258A6D942 ipv6=D5F3FB17504744FB7ECEF46F4B1D155258A6D942 - -# Email sent directly to teor, verified using relay contact info -46.101.151.222:80 orport=443 id=1DBAED235E3957DE1ABD25B4206BE71406FB61F8 -178.62.60.37:80 orport=443 id=175921396C7C426309AB03775A9930B6F611F794 - -# Email sent directly to teor, verified using relay contact info -178.62.197.82:80 orport=443 id=0D3EBA17E1C78F1E9900BABDB23861D46FCAF163 - -# Email sent directly to teor, verified using relay contact info -82.223.21.74:9030 orport=9001 id=7A32C9519D80CA458FC8B034A28F5F6815649A98 ipv6=[2001:470:53e0::cafe]:9050 - -# Email sent directly to teor, verified using relay contact info -146.185.177.103:80 orport=9030 id=9EC5E097663862DF861A18C32B37C5F82284B27D - -# Email sent directly to teor, verified using relay contact info -37.187.22.87:9030 orport=9001 id=36B9E7AC1E36B62A9D6F330ABEB6012BA7F0D400 ipv6=[2001:41d0:a:1657::1]:9001 - -# Email sent directly to teor, verified using relay contact info -37.59.46.159:9030 orport=9001 id=CBD0D1BD110EC52963082D839AC6A89D0AE243E7 - -# Email sent directly to teor, verified using relay contact info -212.47.250.243:9030 orport=9001 id=5B33EDBAEA92F446768B3753549F3B813836D477 -# Confirm with operator before adding these -#163.172.133.36:9030 orport=9001 id=D8C2BD36F01FA86F4401848A0928C4CB7E5FDFF9 -#158.69.216.70:9030 orport=9001 id=0ACE25A978D4422C742D6BC6345896719BF6A7EB - -# Email sent directly to teor, verified using relay contact info -5.199.142.236:9030 orport=9001 id=F4C0EDAA0BF0F7EC138746F8FEF1CE26C7860265 - -# Email sent directly to teor, verified using relay contact info -188.166.133.133:9030 orport=9001 id=774555642FDC1E1D4FDF2E0C31B7CA9501C5C9C7 ipv6=[2a03:b0c0:2:d0::5:f001]:9001 - -# Email sent directly to teor, verified using relay contact info -5.196.88.122:9030 orport=9001 id=0C2C599AFCB26F5CFC2C7592435924C1D63D9484 - -# Email sent directly to teor, verified using relay contact info -46.8.249.10:80 orport=443 id=31670150090A7C3513CB7914B9610E786391A95D - -# Email sent directly to teor, verified using relay contact info -144.76.163.93:9030 orport=9001 id=22F08CF09764C4E8982640D77F71ED72FF26A9AC - -# Email sent directly to teor, verified using relay contact info -46.4.24.161:9030 orport=9001 id=DB4C76A3AD7E234DA0F00D6F1405D8AFDF4D8DED -46.4.24.161:9031 orport=9002 id=7460F3D12EBE861E4EE073F6233047AACFE46AB4 -46.38.51.132:9030 orport=9001 id=810DEFA7E90B6C6C383C063028EC397A71D7214A -163.172.194.53:9030 orport=9001 id=8C00FA7369A7A308F6A137600F0FA07990D9D451 - -# Email sent directly to teor, verified using relay contact info -176.10.107.180:9030 orport=9001 id=3D7E274A87D9A89AF064C13D1EE4CA1F184F2600 -195.154.75.84:9030 orport=9001 id=F80FDE27EFCB3F6A7B4E2CC517133DBFFA78BA2D -195.154.127.246:9030 orport=9001 id=4FEE77AFFD157BBCF2D896AE417FBF647860466C - -# Email sent directly to teor, verified using relay contact info -46.28.207.19:80 orport=443 id=5B92FA5C8A49D46D235735504C72DBB3472BA321 -46.28.207.141:80 orport=443 id=F69BED36177ED727706512BA6A97755025EEA0FB -46.28.205.170:80 orport=443 id=AF322D83A4D2048B22F7F1AF5F38AFF4D09D0B76 -95.183.48.12:80 orport=443 id=7187CED1A3871F837D0E60AC98F374AC541CB0DA - -# Email sent directly to teor, verified using relay contact info -93.180.156.84:9030 orport=9001 id=8844D87E9B038BE3270938F05AF797E1D3C74C0F - -# Email sent directly to teor, verified using relay contact info -37.187.115.157:9030 orport=9001 id=D5039E1EBFD96D9A3F9846BF99EC9F75EDDE902A - -# Email sent directly to teor, verified using relay contact info -5.34.183.205:80 orport=443 id=DDD7871C1B7FA32CB55061E08869A236E61BDDF8 - -# Email sent directly to teor, verified using relay contact info -51.254.246.203:9030 orport=9001 id=47B596B81C9E6277B98623A84B7629798A16E8D5 - -# Email sent directly to teor, verified using relay contact info -5.9.146.203:80 orport=443 id=1F45542A24A61BF9408F1C05E0DCE4E29F2CBA11 - -# Email sent directly to teor, verified using relay contact info -167.114.152.100:9030 orport=443 id=0EF5E5FFC5D1EABCBDA1AFF6F6D6325C5756B0B2 ipv6=[2607:5300:100:200::1608]:443 - -# Email sent directly to teor, verified using relay contact info -192.99.168.102:80 orport=443 id=230A8B2A8BA861210D9B4BA97745AEC217A94207 -167.114.153.21:80 orport=443 id=0B85617241252517E8ECF2CFC7F4C1A32DCD153F - -# Email sent directly to teor, verified using relay contact info -204.11.50.131:9030 orport=9001 id=185F2A57B0C4620582602761097D17DB81654F70 - -# Email sent directly to teor, verified using relay contact info -151.236.222.217:44607 orport=9001 id=94D58704C2589C130C9C39ED148BD8EA468DBA54 - -# Email sent directly to teor, verified using relay contact info -194.150.168.79:11112 orport=11111 id=29F1020B94BE25E6BE1AD13E93CE19D2131B487C - -# Email sent directly to teor, verified using relay contact info -185.35.202.221:9030 orport=9001 id=C13B91384CDD52A871E3ECECE4EF74A7AC7DCB08 ipv6=[2a02:ed06::221]:9001 - -# Email sent directly to teor, verified using relay contact info -5.9.151.241:9030 orport=4223 id=9BF04559224F0F1C3C953D641F1744AF0192543A - -# Email sent directly to teor, verified using relay contact info -89.40.71.149:8081 orport=8080 id=EC639EDAA5121B47DBDF3D6B01A22E48A8CB6CC7 - -# Email sent directly to teor, verified using relay contact info -92.222.20.130:80 orport=443 id=0639612FF149AA19DF3BCEA147E5B8FED6F3C87C - -# Email sent directly to teor, verified using relay contact info -80.112.155.100:9030 orport=9001 id=1163378F239C36CA1BDC730AC50BF4F2976141F5 ipv6=[2001:470:7b02::38]:9001 - -# Email sent directly to teor, verified using relay contact info -83.212.99.68:80 orport=443 id=DDBB2A38252ADDA53E4492DDF982CA6CC6E10EC0 ipv6=[2001:648:2ffc:1225:a800:bff:fe3d:67b5]:443 - -# Email sent directly to teor, verified using relay contact info -95.130.11.147:9030 orport=443 id=6B697F3FF04C26123466A5C0E5D1F8D91925967A - -# Email sent directly to teor, verified using relay contact info -176.31.191.26:9030 orport=9001 id=7350AB9ED7568F22745198359373C04AC783C37C - -# Email sent directly to teor, verified using relay contact info -128.199.55.207:9030 orport=9001 id=BCEF908195805E03E92CCFE669C48738E556B9C5 ipv6=[2a03:b0c0:2:d0::158:3001]:9001 - -# Email sent directly to teor, verified using relay contact info -178.32.216.146:9030 orport=9001 id=17898F9A2EBC7D69DAF87C00A1BD2FABF3C9E1D2 - -# Email sent directly to teor, verified using relay contact info -212.83.40.238:9030 orport=9001 id=F409FA7902FD89270E8DE0D7977EA23BC38E5887 - -# Email sent directly to teor, verified using relay contact info -204.8.156.142:80 orport=443 id=94C4B7B8C50C86A92B6A20107539EE2678CF9A28 - -# Email sent directly to teor, verified using relay contact info -80.240.139.111:80 orport=443 id=DD3BE7382C221F31723C7B294310EF9282B9111B - -# Email sent directly to teor, verified using relay contact info -185.97.32.18:9030 orport=9001 id=3BAB316CAAEC47E71905EB6C65584636D5689A8A - -# Email sent directly to teor, verified using relay contact info -149.56.45.200:9030 orport=9001 id=FE296180018833AF03A8EACD5894A614623D3F76 - -# Email sent directly to teor, verified using relay contact info -81.2.209.10:443 orport=80 id=B6904ADD4C0D10CDA7179E051962350A69A63243 - -# Email sent directly to teor, verified using relay contact info -195.154.164.243:80 orport=443 id=AC66FFA4AB35A59EBBF5BF4C70008BF24D8A7A5C ipv6=[2001:bc8:399f:f000::1]:993 -138.201.26.2:80 orport=443 id=6D3A3ED5671E4E3F58D4951438B10AE552A5FA0F -81.7.16.182:80 orport=443 id=51E1CF613FD6F9F11FE24743C91D6F9981807D82 ipv6=[2a02:180:1:1::517:10b6]:993 -134.119.36.135:80 orport=443 id=763C9556602BD6207771A7A3D958091D44C43228 ipv6=[2a00:1158:3::2a8]:993 -46.228.199.19:80 orport=443 id=E26AFC5F718E21AC502899B20C653AEFF688B0D2 ipv6=[2001:4ba0:cafe:4a::1]:993 -37.200.98.5:80 orport=443 id=231C2B9C8C31C295C472D031E06964834B745996 ipv6=[2a00:1158:3::11a]:993 -46.23.70.195:80 orport=443 id=C9933B3725239B6FAB5227BA33B30BE7B48BB485 -185.15.244.124:80 orport=443 id=935BABE2564F82016C19AEF63C0C40B5753BA3D2 ipv6=[2001:4ba0:cafe:e35::1]:993 -195.154.116.232:80 orport=443 id=B35C5739C8C5AB72094EB2B05738FD1F8EEF6EBD ipv6=[2001:bc8:399f:200::1]:993 -195.154.121.198:80 orport=443 id=0C77421C890D16B6D201283A2244F43DF5BC89DD ipv6=[2001:bc8:399f:100::1]:993 -37.187.20.59:80 orport=443 id=91D23D8A539B83D2FB56AA67ECD4D75CC093AC55 ipv6=[2001:41d0:a:143b::1]:993 -217.12.208.117:80 orport=443 id=E6E18151300F90C235D3809F90B31330737CEB43 ipv6=[2a00:1ca8:a7::1bb]:993 -81.7.10.251:80 orport=443 id=8073670F8F852971298F8AF2C5B23AE012645901 ipv6=[2a02:180:1:1::517:afb]:993 -46.36.39.50:80 orport=443 id=ED4B0DBA79AEF5521564FA0231455DCFDDE73BB6 ipv6=[2a02:25b0:aaaa:aaaa:8d49:b692:4852:0]:995 -91.194.90.103:80 orport=443 id=75C4495F4D80522CA6F6A3FB349F1B009563F4B7 ipv6=[2a02:c200:0:10:3:0:5449:1]:993 -163.172.25.118:80 orport=22 id=0CF8F3E6590F45D50B70F2F7DA6605ECA6CD408F -188.138.88.42:80 orport=443 id=70C55A114C0EF3DC5784A4FAEE64388434A3398F -81.7.13.84:80 orport=443 id=0C1E7DD9ED0676C788933F68A9985ED853CA5812 ipv6=[2a02:180:1:1::5b8f:538c]:993 -213.246.56.95:80 orport=443 id=27E6E8E19C46751E7312420723C6162FF3356A4C ipv6=[2a00:c70:1:213:246:56:95:1]:993 -94.198.100.18:80 orport=443 id=BAACCB29197DB833F107E410E2BFAE5009EE7583 -217.12.203.46:80 orport=443 id=6A29FD8C00D573E6C1D47852345B0E5275BA3307 -212.117.180.107:80 orport=443 id=0B454C7EBA58657B91133A587C1BDAEDC6E23142 -217.12.199.190:80 orport=443 id=A37C47B03FF31CA6937D3D68366B157997FE7BCD ipv6=[2a02:27a8:0:2::486]:993 -216.230.230.247:80 orport=443 id=4C7BF55B1BFF47993DFF995A2926C89C81E4F04A -69.30.215.42:80 orport=443 id=510176C07005D47B23E6796F02C93241A29AA0E9 ipv6=[2604:4300:a:2e::2]:993 -89.46.100.162:80 orport=443 id=6B7191639E179965FD694612C9B2C8FB4267B27D -107.181.174.22:80 orport=443 id=5A551BF2E46BF26CC50A983F7435CB749C752553 ipv6=[2607:f7a0:3:4::4e]:993 - -# Email sent directly to teor, verified using relay contact info -212.238.208.48:9030 orport=9001 id=F406219CDD339026D160E53FCA0EF6857C70F109 ipv6=[2001:984:a8fb:1:ba27:ebff:feac:c109]:9001 - -# Email sent directly to teor, verified using relay contact info -176.158.132.12:9030 orport=9001 id=DC163DDEF4B6F0C6BC226F9F6656A5A30C5C5686 - -# Email sent directly to teor, verified using relay contact info -91.229.20.27:9030 orport=9001 id=9A0D54D3A6D2E0767596BF1515E6162A75B3293F - -# Email sent directly to teor, verified using relay contact info -# Awaiting confirmation of new ORPort from relay operator -80.127.137.19:80 orport=443 id=6EF897645B79B6CB35E853B32506375014DE3621 ipv6=[2001:981:47c1:1::6]:443 - -# Email sent directly to teor, verified using relay contact info -163.172.138.22:80 orport=443 id=8664DC892540F3C789DB37008236C096C871734D - -# Email sent directly to teor, verified using relay contact info -97.74.237.196:9030 orport=9001 id=2F0F32AB1E5B943CA7D062C03F18960C86E70D94 - -# Email sent directly to teor, verified using relay contact info -192.187.124.98:9030 orport=9001 id=FD1871854BFC06D7B02F10742073069F0528B5CC - -# Email sent directly to teor, verified using relay contact info -178.62.98.160:9030 orport=9001 id=8B92044763E880996A988831B15B2B0E5AD1544A - -# Email sent directly to teor, verified using relay contact info -195.154.15.227:9030 orport=9001 id=6C3E3AB2F5F03CD71B637D433BAD924A1ECC5796 - -# Email sent directly to teor, verified using relay contact info -185.100.86.100:80 orport=443 id=0E8C0C8315B66DB5F703804B3889A1DD66C67CE0 - -# Email sent directly to teor, verified using relay contact info -164.132.77.175:9030 orport=9001 id=3B33F6FCA645AD4E91428A3AF7DC736AD9FB727B -78.24.75.53:9030 orport=9001 id=DEB73705B2929AE9BE87091607388939332EF123 - -# Email sent directly to teor, verified using relay contact info -46.101.237.246:9030 orport=9001 id=75F1992FD3F403E9C082A5815EB5D12934CDF46C ipv6=[2a03:b0c0:3:d0::208:5001]:9050 -178.62.86.96:9030 orport=9001 id=439D0447772CB107B886F7782DBC201FA26B92D1 ipv6=[2a03:b0c0:1:d0::3cf:7001]:9050 - -# Email sent directly to teor, verified using relay contact info -91.233.106.121:80 orport=443 id=896364B7996F5DFBA0E15D1A2E06D0B98B555DD6 - -# Email sent directly to teor, verified using relay contact info -167.114.113.48:9030 orport=443 id=2EC0C66EA700C44670444280AABAB1EC78B722A0 - -# Email sent directly to teor, verified using relay contact info -79.120.16.42:9030 orport=9001 id=BD552C165E2ED2887D3F1CCE9CFF155DDA2D86E6 - -# Email sent directly to teor, verified using relay contact info -95.128.43.164:80 orport=443 id=616081EC829593AF4232550DE6FFAA1D75B37A90 ipv6=[2a02:ec0:209:10::4]:443 - -# Email sent directly to teor, verified using relay contact info -166.82.21.200:9030 orport=9029 id=D5C33F3E203728EDF8361EA868B2939CCC43FAFB - -# Email sent directly to teor, verified using relay contact info -91.121.54.8:9030 orport=9001 id=CBEE0F3303C8C50462A12107CA2AE061831931BC - -# Email sent directly to teor, verified using relay contact info -178.217.184.32:8080 orport=443 id=8B7F47AE1A5D954A3E58ACDE0865D09DBA5B738D - -# Email sent directly to teor, verified using relay contact info -85.10.201.47:9030 orport=9001 id=D8B7A3A6542AA54D0946B9DC0257C53B6C376679 ipv6=[2a01:4f8:a0:43eb::beef]:9001 - -# Email sent directly to teor, verified using relay contact info -120.29.217.46:80 orport=443 id=5E853C94AB1F655E9C908924370A0A6707508C62 - -# Email sent directly to teor, verified using relay contact info -37.153.1.10:9030 orport=9001 id=9772EFB535397C942C3AB8804FB35CFFAD012438 - -# Email sent directly to teor, verified using relay contact info -92.222.4.102:9030 orport=9001 id=1A6B8B8272632D8AD38442027F822A367128405C - -# Email sent directly to teor, verified using relay contact info -31.31.78.49:80 orport=443 id=46791D156C9B6C255C2665D4D8393EC7DBAA7798 - -# Email sent directly to teor, verified using relay contact info -96.47.231.214:9030 orport=8080 id=F843CB5729575D76FF1FFBB2179BDCF52C0C6387 -192.99.246.48:9030 orport=9001 id=CD6B149BED1BB254EF6DFF9D75DDB11E7F8A38A4 ipv6=[2607:5300:100:200::de3]:9002 -192.160.102.164:80 orport=9001 id=823AA81E277F366505545522CEDC2F529CE4DC3F ipv6=[2605:e200:d00c:c01d::1111]:9002 - -# Email sent directly to teor, verified using relay contact info -136.243.214.137:80 orport=443 id=B291D30517D23299AD7CEE3E60DFE60D0E3A4664 - -# Email sent directly to teor, verified using relay contact info -192.87.28.28:9030 orport=9001 id=ED2338CAC2711B3E331392E1ED2831219B794024 -192.87.28.82:9030 orport=9001 id=844AE9CAD04325E955E2BE1521563B79FE7094B7 - -# Email sent directly to teor, verified using relay contact info -192.87.28.28:9030 orport=9001 id=ED2338CAC2711B3E331392E1ED2831219B794024 -# OK, but same machine as ED2338CAC2711B3E331392E1ED2831219B794024 -#192.87.28.82:9030 orport=9001 id=844AE9CAD04325E955E2BE1521563B79FE7094B7 - -# https://twitter.com/kosjoli/status/719507270904758272 -85.10.202.87:9030 orport=9001 id=971AFB23C168DCD8EDA17473C1C452B359DE3A5A -176.9.5.116:9030 orport=9001 id=A1EB8D8F1EE28DB98BBB1EAA3B4BEDD303BAB911 -46.4.111.124:9030 orport=9001 id=D9065F9E57899B3D272AA212317AF61A9B14D204 - -# Email sent directly to teor, verified using relay contact info -78.46.164.129:9030 orport=9001 id=52AEA31188331F421B2EDB494DB65CD181E5B257 - -# Email sent directly to teor, verified using relay contact info -185.100.85.61:80 orport=443 id=025B66CEBC070FCB0519D206CF0CF4965C20C96E - -# Email sent directly to teor, verified using relay contact info -108.166.168.158:80 orport=443 id=CDAB3AE06A8C9C6BF817B3B0F1877A4B91465699 - -# Email sent directly to teor, verified using relay contact info -91.219.236.222:80 orport=443 id=EC413181CEB1C8EDC17608BBB177CD5FD8535E99 - -# Email sent directly to teor, verified using relay contact info -185.14.185.240:9030 orport=443 id=D62FB817B0288085FAC38A6DC8B36DCD85B70260 -192.34.63.137:9030 orport=443 id=ABCB4965F1FEE193602B50A365425105C889D3F8 - -# Email sent directly to teor, verified using relay contact info -185.13.38.75:9030 orport=9001 id=D2A1703758A0FBBA026988B92C2F88BAB59F9361 - -# Email sent directly to teor, verified using relay contact info -128.204.39.106:9030 orport=9001 id=6F0F3C09AF9580F7606B34A7678238B3AF7A57B7 - -# Email sent directly to teor, verified using relay contact info -198.50.191.95:80 orport=443 id=39F096961ED2576975C866D450373A9913AFDC92 - -# Email sent directly to teor, verified using relay contact info -167.114.66.61:9696 orport=443 id=DE6CD5F09DF26076F26321B0BDFBE78ACD935C65 ipv6=[2607:5300:100::78d]:443 - -# Email sent directly to teor, verified using relay contact info -66.111.2.20:9030 orport=9001 id=9A68B85A02318F4E7E87F2828039FBD5D75B0142 -66.111.2.16:9030 orport=9001 id=3F092986E9B87D3FDA09B71FA3A602378285C77A - -# Email sent directly to teor, verified using relay contact info -92.222.38.67:80 orport=443 id=DED6892FF89DBD737BA689698A171B2392EB3E82 - -# Email sent directly to teor, verified using relay contact info -212.47.228.115:9030 orport=443 id=BCA017ACDA48330D02BB70716639ED565493E36E - -# Email sent directly to teor, verified using relay contact info -185.100.84.175:80 orport=443 id=39B59AF4FE54FAD8C5085FA9C15FDF23087250DB - -# Email sent directly to teor, verified using relay contact info -166.70.207.2:9030 orport=9001 id=E3DB2E354B883B59E8DC56B3E7A353DDFD457812 - -# Emails sent directly to teor, verified using relay contact info -#69.162.139.9:9030 orport=9001 id=4791FC0692EAB60DF2BCCAFF940B95B74E7654F6 ipv6=[2607:f128:40:1212::45a2:8b09]:9001 - -# Email sent directly to teor, verified using relay contact info -213.239.217.18:1338 orport=1337 id=C37BC191AC389179674578C3E6944E925FE186C2 ipv6=[2a01:4f8:a0:746a:101:1:1:1]:1337 - -# Email sent directly to teor, verified using relay contact info -188.40.128.246:9030 orport=9001 id=AD19490C7DBB26D3A68EFC824F67E69B0A96E601 - -# Email sent directly to teor, verified using relay contact info -88.198.253.13:9030 orport=9001 id=DF924196D69AAE3C00C115A9CCDF7BB62A175310 ipv6=[2a01:4f8:11a:b1f::2]:9001 - -# Email sent directly to teor, verified using relay contact info -185.100.86.128:9030 orport=9001 id=9B31F1F1C1554F9FFB3455911F82E818EF7C7883 -46.36.36.127:9030 orport=9001 id=C80DF89B21FF932DEC0D7821F679B6C79E1449C3 - -# Email sent directly to teor, verified using relay contact info -176.10.104.240:80 orport=443 id=0111BA9B604669E636FFD5B503F382A4B7AD6E80 -176.10.104.240:8080 orport=8443 id=AD86CD1A49573D52A7B6F4A35750F161AAD89C88 -176.10.104.243:80 orport=443 id=88487BDD980BF6E72092EE690E8C51C0AA4A538C -176.10.104.243:8080 orport=8443 id=95DA61AEF23A6C851028C1AA88AD8593F659E60F - -# Email sent directly to teor, verified using relay contact info -107.170.101.39:9030 orport=443 id=30973217E70AF00EBE51797FF6D9AA720A902EAA - -# Email sent directly to teor, verified using relay contact info -192.99.212.139:80 orport=443 id=F10BDE279AE71515DDCCCC61DC19AC8765F8A3CC - -# Email sent directly to teor, verified using relay contact info -163.172.35.249:80 orport=443 id=C08DE49658E5B3CFC6F2A952B453C4B608C9A16A -163.172.35.247:80 orport=443 id=71AB4726D830FAE776D74AEF790CF04D8E0151B4 -163.172.13.124:80 orport=443 id=B771AA877687F88E6F1CA5354756DF6C8A7B6B24 - -# Email sent directly to teor, verified using relay contact info -64.113.32.29:9030 orport=9001 id=30C19B81981F450C402306E2E7CFB6C3F79CB6B2 - -# Email sent directly to teor, verified using relay contact info -212.51.156.193:995 orport=110 id=32E7AAF1F602814D699BEF6761AD03E387758D49 ipv6=[2a02:168:4a01::49]:110 - -# Emails sent directly to teor, verified using relay contact info -51.254.101.242:9002 orport=9001 id=4CC9CC9195EC38645B699A33307058624F660CCF - -# Emails sent directly to teor, verified using relay contact info -85.214.62.48:80 orport=443 id=6A7551EEE18F78A9813096E82BF84F740D32B911 - -# Email sent directly to teor, verified using relay contact info -173.255.245.116:9030 orport=9001 id=91E4015E1F82DAF0121D62267E54A1F661AB6DC7 - -# Email sent directly to teor, verified using relay contact info -62.216.5.120:9030 orport=9001 id=D032D4D617140D6B828FC7C4334860E45E414FBE diff --git a/scripts/maint/format_changelog.py b/scripts/maint/format_changelog.py index e909fc550a..08b2155fa3 100755 --- a/scripts/maint/format_changelog.py +++ b/scripts/maint/format_changelog.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (c) 2014-2015, The Tor Project, Inc. +# Copyright (c) 2014-2019, The Tor Project, Inc. # See LICENSE for licensing information # # This script reformats a section of the changelog to wrap everything to @@ -205,6 +205,8 @@ def head_score(s): score = -300 elif lw.startswith("deprecated version"): score = -200 + elif lw.startswith("directory auth"): + score = -150 elif (('new' in lw and 'requirement' in lw) or ('new' in lw and 'dependenc' in lw) or ('build' in lw and 'requirement' in lw) or @@ -399,7 +401,7 @@ class ChangeLog(object): self.dumpEndOfChangelog() # Let's turn bugs to html. -BUG_PAT = re.compile('(bug|ticket|feature)\s+(\d{4,5})', re.I) +BUG_PAT = re.compile('(bug|ticket|issue|feature)\s+(\d{4,5})', re.I) def bug_html(m): return "%s <a href='https://bugs.torproject.org/%s'>%s</a>" % (m.group(1), m.group(2), m.group(2)) diff --git a/scripts/maint/generate_callgraph.sh b/scripts/maint/generate_callgraph.sh deleted file mode 100755 index c6b33c0aea..0000000000 --- a/scripts/maint/generate_callgraph.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -C_FILES=`echo src/common/*.c src/or/*.c src/tools/*.c` -CFLAGS="-Isrc/ext/trunnel -Isrc/trunnel -I. -Isrc/ext -Isrc/common -DLOCALSTATEDIR=\"\" -DSHARE_DATADIR=\"\" -Dinline=" - -mkdir -p callgraph/src/common -mkdir -p callgraph/src/or -mkdir -p callgraph/src/tools - -for fn in $C_FILES; do - echo $fn - clang $CFLAGS -S -emit-llvm -fno-inline -o - $fn | \ - opt -analyze -print-callgraph >/dev/null 2> "callgraph/${fn}allgraph" -done diff --git a/scripts/maint/lintChanges.py b/scripts/maint/lintChanges.py index b63a4eb3a1..82c118f07e 100755 --- a/scripts/maint/lintChanges.py +++ b/scripts/maint/lintChanges.py @@ -7,7 +7,7 @@ import re import os -KNOWN_GROUPS=set([ +KNOWN_GROUPS = set([ "Minor bugfix", "Minor bugfixes", "Major bugfix", @@ -20,7 +20,50 @@ KNOWN_GROUPS=set([ "Testing", "Documentation", "Code simplification and refactoring", - "Removed features"]) + "Removed features", + "Deprecated features", + "Directory authority changes"]) + +NEEDS_SUBCATEGORIES = set([ + "Minor bugfix", + "Minor bugfixes", + "Major bugfix", + "Major bugfixes", + "Minor feature", + "Minor features", + "Major feature", + "Major features", + ]) + +def split_tor_version(version): + ''' + Return the initial numeric components of the Tor version as a list of ints. + For versions earlier than 0.1.0, returns MAJOR, MINOR, and MICRO. + For versions 0.1.0 and later, returns MAJOR, MINOR, MICRO, and PATCHLEVEL if present. + + If the version is malformed, returns None. + ''' + version_match = re.match('([0-9]+)\.([0-9]+)\.([0-9]+)(\.([0-9]+))?', version) + if version_match is None: + return None + + version_groups = version_match.groups() + if version_groups is None: + return None + if len(version_groups) < 3: + return None + + if len(version_groups) != 5: + return None + version_components = version_groups[0:3] + version_components += version_groups[4:5] + + try: + version_list = [int(v) for v in version_components if v is not None] + except ValueError: + return None + + return version_list def lintfile(fname): have_warned = [] @@ -43,17 +86,13 @@ def lintfile(fname): if bugnum and bugnum not in contents: warn("bug number {} does not appear".format(bugnum)) - lines = contents.split("\n") - m = re.match(r'^[ ]{2}o ([^\(:]*)([^:]*):', contents) if not m: - warn("header not in format expected") + warn("Header not in format expected. (' o Foo:' or ' o Foo (Bar):')") elif m.group(1).strip() not in KNOWN_GROUPS: - warn("Weird header: %r"%m.group(1)) - elif ( ("bugfix" in m.group(1) or "feature" in m.group(1)) and - ("Removed" not in m.group(1)) and - '(' not in m.group(2)): - warn("Missing subcategory on %s"%m.group(1)) + warn("Unrecognized header: %r" % m.group(1)) + elif (m.group(1) in NEEDS_SUBCATEGORIES and '(' not in m.group(2)): + warn("Missing subcategory on %r" % m.group(1)) if m: isBug = ("bug" in m.group(1).lower() or "fix" in m.group(1).lower()) @@ -63,23 +102,72 @@ def lintfile(fname): contents = " ".join(contents.split()) if re.search(r'\#\d{2,}', contents): - warn("don't use a # before ticket numbers") + warn("Don't use a # before ticket numbers. ('bug 1234' not '#1234')") if isBug and not re.search(r'(\d+)', contents): - warn("bugfix does not mention a number") - elif isBug and not re.search(r'Fixes ([a-z ]*)bug (\d+)', contents): - warn("bugfix does not say 'Fixes bug XXX'") + warn("Ticket marked as bugfix, but does not mention a number.") + elif isBug and not re.search(r'Fixes ([a-z ]*)bugs? (\d+)', contents): + warn("Ticket marked as bugfix, but does not say 'Fixes bug XXX'") if re.search(r'[bB]ug (\d+)', contents): if not re.search(r'[Bb]ugfix on ', contents): - warn("bugfix does not say 'bugfix on X.Y.Z'") - elif not re.search('[fF]ixes ([a-z ]*)bug (\d+); bugfix on ', + warn("Bugfix does not say 'bugfix on X.Y.Z'") + elif not re.search('[fF]ixes ([a-z ]*)bugs? (\d+)((, \d+)* and \d+)?; bugfix on ', contents): - warn("bugfix incant is not semicoloned") - + warn("Bugfix does not say 'Fixes bug X; bugfix on Y'") + elif re.search('tor-([0-9]+)', contents): + warn("Do not prefix versions with 'tor-'. ('0.1.2', not 'tor-0.1.2'.)") + else: + bugfix_match = re.search('bugfix on ([0-9]+\.[0-9]+\.[0-9]+)', contents) + if bugfix_match is None: + warn("Versions must have at least 3 digits. ('0.1.2', '0.3.4.8', or '0.3.5.1-alpha'.)") + elif bugfix_match.group(0) is None: + warn("Versions must have at least 3 digits. ('0.1.2', '0.3.4.8', or '0.3.5.1-alpha'.)") + else: + bugfix_match = re.search('bugfix on ([0-9a-z][-.0-9a-z]+[0-9a-z])', contents) + bugfix_group = bugfix_match.groups() if bugfix_match is not None else None + bugfix_version = bugfix_group[0] if bugfix_group is not None else None + package_version = os.environ.get('PACKAGE_VERSION', None) + if bugfix_version is None: + # This should be unreachable, unless the patterns are out of sync + warn("Malformed bugfix version.") + elif package_version is not None: + # If $PACKAGE_VERSION isn't set, skip this check + bugfix_split = split_tor_version(bugfix_version) + package_split = split_tor_version(package_version) + if bugfix_split is None: + # This should be unreachable, unless the patterns are out of sync + warn("Malformed bugfix version: '{}'.".format(bugfix_version)) + elif package_split is None: + # This should be unreachable, unless the patterns are out of sync, or the package versioning scheme has changed + warn("Malformed $PACKAGE_VERSION: '{}'.".format(package_version)) + elif bugfix_split > package_split: + warn("Bugfixes must be made on earlier versions (or this version). (Bugfix on version: '{}', current tor package version: '{}'.)".format(bugfix_version, package_version)) + + return have_warned != [] + +def files(args): + """Walk through the arguments: for directories, yield their contents; + for files, just yield the files. Only search one level deep, because + that's how the changes directory is laid out.""" + for f in args: + if os.path.isdir(f): + for item in os.listdir(f): + if item.startswith("."): #ignore dotfiles + continue + yield os.path.join(f, item) + else: + yield f if __name__ == '__main__': - for fname in sys.argv[1:]: + problems = 0 + for fname in files(sys.argv[1:]): if fname.endswith("~"): continue - lintfile(fname) + if lintfile(fname): + problems += 1 + + if problems: + sys.exit(1) + else: + sys.exit(0) diff --git a/scripts/maint/pre-commit.git-hook b/scripts/maint/pre-commit.git-hook new file mode 100755 index 0000000000..b4c4ce2061 --- /dev/null +++ b/scripts/maint/pre-commit.git-hook @@ -0,0 +1,26 @@ +#!/bin/bash +# +# To install this script, copy it to .git/hooks/pre-commit in local copy of +# tor git repo and make sure it has permission to execute. +# +# This is pre-commit git hook script that prevents commiting your changeset if +# it fails our code formatting or changelog entry formatting checkers. + +workdir=$(git rev-parse --show-toplevel) + +cd "$workdir" || exit 1 + +python scripts/maint/lintChanges.py ./changes/* + +perl scripts/maint/checkSpace.pl -C \ +src/lib/*/*.[ch] \ +src/core/*/*.[ch] \ +src/feature/*/*.[ch] \ +src/app/*/*.[ch] \ +src/test/*.[ch] \ +src/test/*/*.[ch] \ +src/tools/*.[ch] + +if test -e scripts/maint/checkIncludes.py; then + python scripts/maint/checkIncludes.py +fi diff --git a/scripts/maint/pre-push.git-hook b/scripts/maint/pre-push.git-hook new file mode 100755 index 0000000000..26296023fb --- /dev/null +++ b/scripts/maint/pre-push.git-hook @@ -0,0 +1,61 @@ +#!/bin/bash + +# To install this script, copy it into .git/hooks/pre-push path in your +# local copy of git repository. Make sure it has permission to execute. +# +# This is git pre-push hook script to prevent "fixup!" and "squash!" commits +# from ending up in upstream branches (master, release-* or maint-*). +# +# The following sample script was used as starting point: +# https://github.com/git/git/blob/master/templates/hooks--pre-push.sample + +z40=0000000000000000000000000000000000000000 + +CUR_BRANCH=$(git rev-parse --abbrev-ref HEAD) +if [ "$CUR_BRANCH" != "master" ] && [[ $CUR_BRANCH != release-* ]] && + [[ $CUR_BRANCH != maint-* ]] +then + exit 0 +fi + +echo "Running pre-push hook" + +# shellcheck disable=SC2034 +while read -r local_ref local_sha remote_ref remote_sha +do + if [ "$local_sha" = $z40 ] + then + # Handle delete + : + else + if [ "$remote_sha" = $z40 ] + then + # New branch, examine all commits + range="$local_sha" + else + # Update to existing branch, examine new commits + range="$remote_sha..$local_sha" + fi + + # Check for fixup! commit + commit=$(git rev-list -n 1 --grep '^fixup!' "$range") + if [ -n "$commit" ] + then + echo >&2 "Found fixup! commit in $local_ref, not pushing" + echo >&2 "If you really want to push this, use --no-verify." + exit 1 + fi + + # Check for squash! commit + commit=$(git rev-list -n 1 --grep '^squash!' "$range") + if [ -n "$commit" ] + then + echo >&2 "Found squash! commit in $local_ref, not pushing" + echo >&2 "If you really want to push this, use --no-verify." + exit 1 + fi + fi +done + +exit 0 + diff --git a/scripts/maint/rectify_include_paths.py b/scripts/maint/rectify_include_paths.py new file mode 100755 index 0000000000..401fadae6d --- /dev/null +++ b/scripts/maint/rectify_include_paths.py @@ -0,0 +1,60 @@ +#!/usr/bin/python3 + +import os +import os.path +import re + +# Find all the include files, map them to their real names. + +def exclude(paths, dirnames): + for p in paths: + if p in dirnames: + dirnames.remove(p) + +def get_include_map(): + includes = { } + + for dirpath,dirnames,fnames in os.walk("src"): + exclude(["ext", "win32"], dirnames) + + for fname in fnames: + if fname.endswith(".h"): + assert fname not in includes + include = os.path.join(dirpath, fname) + assert include.startswith("src/") + includes[fname] = include[4:] + + return includes + +INCLUDE_PAT = re.compile(r'( *# *include +")([^"]+)(".*)') + +def get_base_header_name(hdr): + return os.path.split(hdr)[1] + +def fix_includes(inp, out, mapping): + for line in inp: + m = INCLUDE_PAT.match(line) + if m: + include,hdr,rest = m.groups() + basehdr = get_base_header_name(hdr) + if basehdr in mapping: + out.write('{}{}{}\n'.format(include,mapping[basehdr],rest)) + continue + + out.write(line) + +incs = get_include_map() + +for dirpath,dirnames,fnames in os.walk("src"): + exclude(["trunnel"], dirnames) + + for fname in fnames: + if fname.endswith(".c") or fname.endswith(".h"): + fname = os.path.join(dirpath, fname) + tmpfile = fname+".tmp" + f_in = open(fname, 'r') + f_out = open(tmpfile, 'w') + fix_includes(f_in, f_out, incs) + f_in.close() + f_out.close() + os.rename(tmpfile, fname) diff --git a/scripts/maint/redox.py b/scripts/maint/redox.py index 43f5b6eb16..203cce0107 100755 --- a/scripts/maint/redox.py +++ b/scripts/maint/redox.py @@ -1,6 +1,6 @@ #!/usr/bin/python # -# Copyright (c) 2008-2015, The Tor Project, Inc. +# Copyright (c) 2008-2019, The Tor Project, Inc. # See LICENSE for licensing information. # # Hi! @@ -101,7 +101,7 @@ def read(): def findline(lines, lineno, ident): """Given a list of all the lines in the file (adjusted so 1-indexing works), - a line number that ident is alledgedly on, and ident, I figure out + a line number that ident is allegedly on, and ident, I figure out the line where ident was really declared.""" lno = lineno for lineno in xrange(lineno, 0, -1): diff --git a/scripts/maint/run_calltool.sh b/scripts/maint/run_calltool.sh new file mode 100755 index 0000000000..b0268322f4 --- /dev/null +++ b/scripts/maint/run_calltool.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +# You can find calltool at https://gitweb.torproject.org/user/nickm/calltool.git + +set -e + +if test "x$CALLTOOL_PATH" != "x"; then + PYTHONPATH="${CALLTOOL_PATH}:${PYTHONPATH}" + export PYTHONPATH +fi + +mkdir -p callgraph + +SUBITEMS="fn_graph fn_invgraph fn_scc fn_scc_weaklinks module_graph module_invgraph module_scc module_scc_weaklinks" + +for calculation in $SUBITEMS; do + echo "======== $calculation" + python -m calltool "$calculation" > callgraph/"$calculation" +done + +cat <<EOF > callgraph/README +This directory holds output from calltool, as run on Tor. For more +information about each of these files, see the NOTES and README files in +the calltool distribution. + +You can find calltool at + https://gitweb.torproject.org/user/nickm/calltool.git +EOF + diff --git a/scripts/maint/sortChanges.py b/scripts/maint/sortChanges.py index d6ec0e269d..986b94b025 100755 --- a/scripts/maint/sortChanges.py +++ b/scripts/maint/sortChanges.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (c) 2014-2015, The Tor Project, Inc. +# Copyright (c) 2014-2019, The Tor Project, Inc. # See LICENSE for licensing information """This script sorts a bunch of changes files listed on its command diff --git a/scripts/maint/updateCopyright.pl b/scripts/maint/updateCopyright.pl index 8bd6a18210..36894b1baf 100755 --- a/scripts/maint/updateCopyright.pl +++ b/scripts/maint/updateCopyright.pl @@ -1,7 +1,7 @@ #!/usr/bin/perl -i -w -p -$NEWYEAR=2016; +$NEWYEAR=2019; -s/Copyright(.*) (201[^6]), The Tor Project/Copyright$1 $2-${NEWYEAR}, The Tor Project/; +s/Copyright(.*) (201[^9]), The Tor Project/Copyright$1 $2-${NEWYEAR}, The Tor Project/; s/Copyright(.*)-(20..), The Tor Project/Copyright$1-${NEWYEAR}, The Tor Project/; diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py deleted file mode 100755 index 110ecda64c..0000000000 --- a/scripts/maint/updateFallbackDirs.py +++ /dev/null @@ -1,1999 +0,0 @@ -#!/usr/bin/python - -# Usage: scripts/maint/updateFallbackDirs.py > src/or/fallback_dirs.inc -# -# This script should be run from a stable, reliable network connection, -# with no other network activity (and not over tor). -# If this is not possible, please disable: -# PERFORM_IPV4_DIRPORT_CHECKS and PERFORM_IPV6_DIRPORT_CHECKS -# -# Needs dateutil (and potentially other python packages) -# Needs stem available in your PYTHONPATH, or just ln -s ../stem/stem . -# Optionally uses ipaddress (python 3 builtin) or py2-ipaddress (package) -# for netblock analysis, in PYTHONPATH, or just -# ln -s ../py2-ipaddress-3.4.1/ipaddress.py . -# -# Then read the logs to make sure the fallbacks aren't dominated by a single -# netblock or port - -# Script by weasel, April 2015 -# Portions by gsathya & karsten, 2013 -# https://trac.torproject.org/projects/tor/attachment/ticket/8374/dir_list.2.py -# Modifications by teor, 2015 - -import StringIO -import string -import re -import datetime -import gzip -import os.path -import json -import math -import sys -import urllib -import urllib2 -import hashlib -import dateutil.parser -# bson_lazy provides bson -#from bson import json_util -import copy - -from stem.descriptor.remote import DescriptorDownloader - -import logging -# INFO tells you why each relay was included or excluded -# WARN tells you about potential misconfigurations and relay detail changes -logging.basicConfig(level=logging.WARNING) -logging.root.name = '' -# INFO tells you about each consensus download attempt -logging.getLogger('stem').setLevel(logging.WARNING) - -HAVE_IPADDRESS = False -try: - # python 3 builtin, or install package py2-ipaddress - # there are several ipaddress implementations for python 2 - # with slightly different semantics with str typed text - # fortunately, all our IP addresses are in unicode - import ipaddress - HAVE_IPADDRESS = True -except ImportError: - # if this happens, we avoid doing netblock analysis - logging.warning('Unable to import ipaddress, please install py2-ipaddress.' + - ' A fallback list will be created, but optional netblock' + - ' analysis will not be performed.') - -## Top-Level Configuration - -# Output all candidate fallbacks, or only output selected fallbacks? -OUTPUT_CANDIDATES = False - -# Perform DirPort checks over IPv4? -# Change this to False if IPv4 doesn't work for you, or if you don't want to -# download a consensus for each fallback -# Don't check ~1000 candidates when OUTPUT_CANDIDATES is True -PERFORM_IPV4_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else True - -# Perform DirPort checks over IPv6? -# If you know IPv6 works for you, set this to True -# This will exclude IPv6 relays without an IPv6 DirPort configured -# So it's best left at False until #18394 is implemented -# Don't check ~1000 candidates when OUTPUT_CANDIDATES is True -PERFORM_IPV6_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else False - -# Output fallback name, flags, and ContactInfo in a C comment? -OUTPUT_COMMENTS = True if OUTPUT_CANDIDATES else False - -# Output matching ContactInfo in fallbacks list or the blacklist? -# Useful if you're trying to contact operators -CONTACT_COUNT = True if OUTPUT_CANDIDATES else False -CONTACT_BLACKLIST_COUNT = True if OUTPUT_CANDIDATES else False - -## OnionOO Settings - -ONIONOO = 'https://onionoo.torproject.org/' -#ONIONOO = 'https://onionoo.thecthulhu.com/' - -# Don't bother going out to the Internet, just use the files available locally, -# even if they're very old -LOCAL_FILES_ONLY = False - -## Whitelist / Blacklist Filter Settings - -# The whitelist contains entries that are included if all attributes match -# (IPv4, dirport, orport, id, and optionally IPv6 and IPv6 orport) -# The blacklist contains (partial) entries that are excluded if any -# sufficiently specific group of attributes matches: -# IPv4 & DirPort -# IPv4 & ORPort -# ID -# IPv6 & DirPort -# IPv6 & IPv6 ORPort -# If neither port is included in the blacklist, the entire IP address is -# blacklisted. - -# What happens to entries in neither list? -# When True, they are included, when False, they are excluded -INCLUDE_UNLISTED_ENTRIES = True if OUTPUT_CANDIDATES else False - -# If an entry is in both lists, what happens? -# When True, it is excluded, when False, it is included -BLACKLIST_EXCLUDES_WHITELIST_ENTRIES = True - -WHITELIST_FILE_NAME = 'scripts/maint/fallback.whitelist' -BLACKLIST_FILE_NAME = 'scripts/maint/fallback.blacklist' - -# The number of bytes we'll read from a filter file before giving up -MAX_LIST_FILE_SIZE = 1024 * 1024 - -## Eligibility Settings - -# Reduced due to a bug in tor where a relay submits a 0 DirPort when restarted -# This causes OnionOO to (correctly) reset its stability timer -# This issue will be fixed in 0.2.7.7 and 0.2.8.2 -# Until then, the CUTOFFs below ensure a decent level of stability. -ADDRESS_AND_PORT_STABLE_DAYS = 7 -# What time-weighted-fraction of these flags must FallbackDirs -# Equal or Exceed? -CUTOFF_RUNNING = .95 -CUTOFF_V2DIR = .95 -CUTOFF_GUARD = .95 -# What time-weighted-fraction of these flags must FallbackDirs -# Equal or Fall Under? -# .00 means no bad exits -PERMITTED_BADEXIT = .00 - -# older entries' weights are adjusted with ALPHA^(age in days) -AGE_ALPHA = 0.99 - -# this factor is used to scale OnionOO entries to [0,1] -ONIONOO_SCALE_ONE = 999. - -## Fallback Count Limits - -# The target for these parameters is 20% of the guards in the network -# This is around 200 as of October 2015 -_FB_POG = 0.2 -FALLBACK_PROPORTION_OF_GUARDS = None if OUTPUT_CANDIDATES else _FB_POG - -# We want exactly 100 fallbacks for the initial release -# This gives us scope to add extra fallbacks to the list as needed -# Limit the number of fallbacks (eliminating lowest by advertised bandwidth) -MAX_FALLBACK_COUNT = None if OUTPUT_CANDIDATES else 100 -# Emit a C #error if the number of fallbacks is below -MIN_FALLBACK_COUNT = 100 - -## Fallback Bandwidth Requirements - -# Any fallback with the Exit flag has its bandwidth multipled by this fraction -# to make sure we aren't further overloading exits -# (Set to 1.0, because we asked that only lightly loaded exits opt-in, -# and the extra load really isn't that much for large relays.) -EXIT_BANDWIDTH_FRACTION = 1.0 - -# If a single fallback's bandwidth is too low, it's pointless adding it -# We expect fallbacks to handle an extra 30 kilobytes per second of traffic -# Make sure they can support a hundred times the expected extra load -# (Use 102.4 to make it come out nicely in MB/s) -# We convert this to a consensus weight before applying the filter, -# because all the bandwidth amounts are specified by the relay -MIN_BANDWIDTH = 102.4 * 30.0 * 1024.0 - -# Clients will time out after 30 seconds trying to download a consensus -# So allow fallback directories half that to deliver a consensus -# The exact download times might change based on the network connection -# running this script, but only by a few seconds -# There is also about a second of python overhead -CONSENSUS_DOWNLOAD_SPEED_MAX = 15.0 -# If the relay fails a consensus check, retry the download -# This avoids delisting a relay due to transient network conditions -CONSENSUS_DOWNLOAD_RETRY = True - -## Fallback Weights for Client Selection - -# All fallback weights are equal, and set to the value below -# Authorities are weighted 1.0 by default -# Clients use these weights to select fallbacks and authorities at random -# If there are 100 fallbacks and 9 authorities: -# - each fallback is chosen with probability 10.0/(10.0*100 + 1.0*9) ~= 0.99% -# - each authority is chosen with probability 1.0/(10.0*100 + 1.0*9) ~= 0.09% -# A client choosing a bootstrap directory server will choose a fallback for -# 10.0/(10.0*100 + 1.0*9) * 100 = 99.1% of attempts, and an authority for -# 1.0/(10.0*100 + 1.0*9) * 9 = 0.9% of attempts. -# (This disregards the bootstrap schedules, where clients start by choosing -# from fallbacks & authoritites, then later choose from only authorities.) -FALLBACK_OUTPUT_WEIGHT = 10.0 - -## Parsing Functions - -def parse_ts(t): - return datetime.datetime.strptime(t, "%Y-%m-%d %H:%M:%S") - -def remove_bad_chars(raw_string, bad_char_list): - # Remove each character in the bad_char_list - cleansed_string = raw_string - for c in bad_char_list: - cleansed_string = cleansed_string.replace(c, '') - return cleansed_string - -def cleanse_unprintable(raw_string): - # Remove all unprintable characters - cleansed_string = '' - for c in raw_string: - if c in string.printable: - cleansed_string += c - return cleansed_string - -def cleanse_whitespace(raw_string): - # Replace all whitespace characters with a space - cleansed_string = raw_string - for c in string.whitespace: - cleansed_string = cleansed_string.replace(c, ' ') - return cleansed_string - -def cleanse_c_multiline_comment(raw_string): - cleansed_string = raw_string - # Embedded newlines should be removed by tor/onionoo, but let's be paranoid - cleansed_string = cleanse_whitespace(cleansed_string) - # ContactInfo and Version can be arbitrary binary data - cleansed_string = cleanse_unprintable(cleansed_string) - # Prevent a malicious / unanticipated string from breaking out - # of a C-style multiline comment - # This removes '/*' and '*/' and '//' - bad_char_list = '*/' - # Prevent a malicious string from using C nulls - bad_char_list += '\0' - # Be safer by removing bad characters entirely - cleansed_string = remove_bad_chars(cleansed_string, bad_char_list) - # Some compilers may further process the content of comments - # There isn't much we can do to cover every possible case - # But comment-based directives are typically only advisory - return cleansed_string - -def cleanse_c_string(raw_string): - cleansed_string = raw_string - # Embedded newlines should be removed by tor/onionoo, but let's be paranoid - cleansed_string = cleanse_whitespace(cleansed_string) - # ContactInfo and Version can be arbitrary binary data - cleansed_string = cleanse_unprintable(cleansed_string) - # Prevent a malicious address/fingerprint string from breaking out - # of a C-style string - bad_char_list = '"' - # Prevent a malicious string from using escapes - bad_char_list += '\\' - # Prevent a malicious string from using C nulls - bad_char_list += '\0' - # Be safer by removing bad characters entirely - cleansed_string = remove_bad_chars(cleansed_string, bad_char_list) - # Some compilers may further process the content of strings - # There isn't much we can do to cover every possible case - # But this typically only results in changes to the string data - return cleansed_string - -## OnionOO Source Functions - -# a dictionary of source metadata for each onionoo query we've made -fetch_source = {} - -# register source metadata for 'what' -# assumes we only retrieve one document for each 'what' -def register_fetch_source(what, url, relays_published, version): - fetch_source[what] = {} - fetch_source[what]['url'] = url - fetch_source[what]['relays_published'] = relays_published - fetch_source[what]['version'] = version - -# list each registered source's 'what' -def fetch_source_list(): - return sorted(fetch_source.keys()) - -# given 'what', provide a multiline C comment describing the source -def describe_fetch_source(what): - desc = '/*' - desc += '\n' - desc += 'Onionoo Source: ' - desc += cleanse_c_multiline_comment(what) - desc += ' Date: ' - desc += cleanse_c_multiline_comment(fetch_source[what]['relays_published']) - desc += ' Version: ' - desc += cleanse_c_multiline_comment(fetch_source[what]['version']) - desc += '\n' - desc += 'URL: ' - desc += cleanse_c_multiline_comment(fetch_source[what]['url']) - desc += '\n' - desc += '*/' - return desc - -## File Processing Functions - -def write_to_file(str, file_name, max_len): - try: - with open(file_name, 'w') as f: - f.write(str[0:max_len]) - except EnvironmentError, error: - logging.error('Writing file %s failed: %d: %s'% - (file_name, - error.errno, - error.strerror) - ) - -def read_from_file(file_name, max_len): - try: - if os.path.isfile(file_name): - with open(file_name, 'r') as f: - return f.read(max_len) - except EnvironmentError, error: - logging.info('Loading file %s failed: %d: %s'% - (file_name, - error.errno, - error.strerror) - ) - return None - -def load_possibly_compressed_response_json(response): - if response.info().get('Content-Encoding') == 'gzip': - buf = StringIO.StringIO( response.read() ) - f = gzip.GzipFile(fileobj=buf) - return json.load(f) - else: - return json.load(response) - -def load_json_from_file(json_file_name): - # An exception here may be resolved by deleting the .last_modified - # and .json files, and re-running the script - try: - with open(json_file_name, 'r') as f: - return json.load(f) - except EnvironmentError, error: - raise Exception('Reading not-modified json file %s failed: %d: %s'% - (json_file_name, - error.errno, - error.strerror) - ) - -## OnionOO Functions - -def datestr_to_datetime(datestr): - # Parse datetimes like: Fri, 02 Oct 2015 13:34:14 GMT - if datestr is not None: - dt = dateutil.parser.parse(datestr) - else: - # Never modified - use start of epoch - dt = datetime.datetime.utcfromtimestamp(0) - # strip any timezone out (in case they're supported in future) - dt = dt.replace(tzinfo=None) - return dt - -def onionoo_fetch(what, **kwargs): - params = kwargs - params['type'] = 'relay' - #params['limit'] = 10 - params['first_seen_days'] = '%d-'%(ADDRESS_AND_PORT_STABLE_DAYS,) - params['last_seen_days'] = '-7' - params['flag'] = 'V2Dir' - url = ONIONOO + what + '?' + urllib.urlencode(params) - - # Unfortunately, the URL is too long for some OS filenames, - # but we still don't want to get files from different URLs mixed up - base_file_name = what + '-' + hashlib.sha1(url).hexdigest() - - full_url_file_name = base_file_name + '.full_url' - MAX_FULL_URL_LENGTH = 1024 - - last_modified_file_name = base_file_name + '.last_modified' - MAX_LAST_MODIFIED_LENGTH = 64 - - json_file_name = base_file_name + '.json' - - if LOCAL_FILES_ONLY: - # Read from the local file, don't write to anything - response_json = load_json_from_file(json_file_name) - else: - # store the full URL to a file for debugging - # no need to compare as long as you trust SHA-1 - write_to_file(url, full_url_file_name, MAX_FULL_URL_LENGTH) - - request = urllib2.Request(url) - request.add_header('Accept-encoding', 'gzip') - - # load the last modified date from the file, if it exists - last_mod_date = read_from_file(last_modified_file_name, - MAX_LAST_MODIFIED_LENGTH) - if last_mod_date is not None: - request.add_header('If-modified-since', last_mod_date) - - # Parse last modified date - last_mod = datestr_to_datetime(last_mod_date) - - # Not Modified and still recent enough to be useful - # Onionoo / Globe used to use 6 hours, but we can afford a day - required_freshness = datetime.datetime.utcnow() - # strip any timezone out (to match dateutil.parser) - required_freshness = required_freshness.replace(tzinfo=None) - required_freshness -= datetime.timedelta(hours=24) - - # Make the OnionOO request - response_code = 0 - try: - response = urllib2.urlopen(request) - response_code = response.getcode() - except urllib2.HTTPError, error: - response_code = error.code - if response_code == 304: # not modified - pass - else: - raise Exception("Could not get " + url + ": " - + str(error.code) + ": " + error.reason) - - if response_code == 200: # OK - last_mod = datestr_to_datetime(response.info().get('Last-Modified')) - - # Check for freshness - if last_mod < required_freshness: - if last_mod_date is not None: - # This check sometimes fails transiently, retry the script if it does - date_message = "Outdated data: last updated " + last_mod_date - else: - date_message = "No data: never downloaded " - raise Exception(date_message + " from " + url) - - # Process the data - if response_code == 200: # OK - - response_json = load_possibly_compressed_response_json(response) - - with open(json_file_name, 'w') as f: - # use the most compact json representation to save space - json.dump(response_json, f, separators=(',',':')) - - # store the last modified date in its own file - if response.info().get('Last-modified') is not None: - write_to_file(response.info().get('Last-Modified'), - last_modified_file_name, - MAX_LAST_MODIFIED_LENGTH) - - elif response_code == 304: # Not Modified - - response_json = load_json_from_file(json_file_name) - - else: # Unexpected HTTP response code not covered in the HTTPError above - raise Exception("Unexpected HTTP response code to " + url + ": " - + str(response_code)) - - register_fetch_source(what, - url, - response_json['relays_published'], - response_json['version']) - - return response_json - -def fetch(what, **kwargs): - #x = onionoo_fetch(what, **kwargs) - # don't use sort_keys, as the order of or_addresses is significant - #print json.dumps(x, indent=4, separators=(',', ': ')) - #sys.exit(0) - - return onionoo_fetch(what, **kwargs) - -## Fallback Candidate Class - -class Candidate(object): - CUTOFF_ADDRESS_AND_PORT_STABLE = (datetime.datetime.utcnow() - - datetime.timedelta(ADDRESS_AND_PORT_STABLE_DAYS)) - - def __init__(self, details): - for f in ['fingerprint', 'nickname', 'last_changed_address_or_port', - 'consensus_weight', 'or_addresses', 'dir_address']: - if not f in details: raise Exception("Document has no %s field."%(f,)) - - if not 'contact' in details: - details['contact'] = None - if not 'flags' in details or details['flags'] is None: - details['flags'] = [] - if (not 'advertised_bandwidth' in details - or details['advertised_bandwidth'] is None): - # relays without advertised bandwdith have it calculated from their - # consensus weight - details['advertised_bandwidth'] = 0 - if (not 'effective_family' in details - or details['effective_family'] is None): - details['effective_family'] = [] - details['last_changed_address_or_port'] = parse_ts( - details['last_changed_address_or_port']) - self._data = details - self._stable_sort_or_addresses() - - self._fpr = self._data['fingerprint'] - self._running = self._guard = self._v2dir = 0. - self._split_dirport() - self._compute_orport() - if self.orport is None: - raise Exception("Failed to get an orport for %s."%(self._fpr,)) - self._compute_ipv6addr() - if not self.has_ipv6(): - logging.debug("Failed to get an ipv6 address for %s."%(self._fpr,)) - - def _stable_sort_or_addresses(self): - # replace self._data['or_addresses'] with a stable ordering, - # sorting the secondary addresses in string order - # leave the received order in self._data['or_addresses_raw'] - self._data['or_addresses_raw'] = self._data['or_addresses'] - or_address_primary = self._data['or_addresses'][:1] - # subsequent entries in the or_addresses array are in an arbitrary order - # so we stabilise the addresses by sorting them in string order - or_addresses_secondaries_stable = sorted(self._data['or_addresses'][1:]) - or_addresses_stable = or_address_primary + or_addresses_secondaries_stable - self._data['or_addresses'] = or_addresses_stable - - def get_fingerprint(self): - return self._fpr - - # is_valid_ipv[46]_address by gsathya, karsten, 2013 - @staticmethod - def is_valid_ipv4_address(address): - if not isinstance(address, (str, unicode)): - return False - - # check if there are four period separated values - if address.count(".") != 3: - return False - - # checks that each value in the octet are decimal values between 0-255 - for entry in address.split("."): - if not entry.isdigit() or int(entry) < 0 or int(entry) > 255: - return False - elif entry[0] == "0" and len(entry) > 1: - return False # leading zeros, for instance in "1.2.3.001" - - return True - - @staticmethod - def is_valid_ipv6_address(address): - if not isinstance(address, (str, unicode)): - return False - - # remove brackets - address = address[1:-1] - - # addresses are made up of eight colon separated groups of four hex digits - # with leading zeros being optional - # https://en.wikipedia.org/wiki/IPv6#Address_format - - colon_count = address.count(":") - - if colon_count > 7: - return False # too many groups - elif colon_count != 7 and not "::" in address: - return False # not enough groups and none are collapsed - elif address.count("::") > 1 or ":::" in address: - return False # multiple groupings of zeros can't be collapsed - - found_ipv4_on_previous_entry = False - for entry in address.split(":"): - # If an IPv6 address has an embedded IPv4 address, - # it must be the last entry - if found_ipv4_on_previous_entry: - return False - if not re.match("^[0-9a-fA-f]{0,4}$", entry): - if not Candidate.is_valid_ipv4_address(entry): - return False - else: - found_ipv4_on_previous_entry = True - - return True - - def _split_dirport(self): - # Split the dir_address into dirip and dirport - (self.dirip, _dirport) = self._data['dir_address'].split(':', 2) - self.dirport = int(_dirport) - - def _compute_orport(self): - # Choose the first ORPort that's on the same IPv4 address as the DirPort. - # In rare circumstances, this might not be the primary ORPort address. - # However, _stable_sort_or_addresses() ensures we choose the same one - # every time, even if onionoo changes the order of the secondaries. - self._split_dirport() - self.orport = None - for i in self._data['or_addresses']: - if i != self._data['or_addresses'][0]: - logging.debug('Secondary IPv4 Address Used for %s: %s'%(self._fpr, i)) - (ipaddr, port) = i.rsplit(':', 1) - if (ipaddr == self.dirip) and Candidate.is_valid_ipv4_address(ipaddr): - self.orport = int(port) - return - - def _compute_ipv6addr(self): - # Choose the first IPv6 address that uses the same port as the ORPort - # Or, choose the first IPv6 address in the list - # _stable_sort_or_addresses() ensures we choose the same IPv6 address - # every time, even if onionoo changes the order of the secondaries. - self.ipv6addr = None - self.ipv6orport = None - # Choose the first IPv6 address that uses the same port as the ORPort - for i in self._data['or_addresses']: - (ipaddr, port) = i.rsplit(':', 1) - if (port == self.orport) and Candidate.is_valid_ipv6_address(ipaddr): - self.ipv6addr = ipaddr - self.ipv6orport = int(port) - return - # Choose the first IPv6 address in the list - for i in self._data['or_addresses']: - (ipaddr, port) = i.rsplit(':', 1) - if Candidate.is_valid_ipv6_address(ipaddr): - self.ipv6addr = ipaddr - self.ipv6orport = int(port) - return - - @staticmethod - def _extract_generic_history(history, which='unknown'): - # given a tree like this: - # { - # "1_month": { - # "count": 187, - # "factor": 0.001001001001001001, - # "first": "2015-02-27 06:00:00", - # "interval": 14400, - # "last": "2015-03-30 06:00:00", - # "values": [ - # 999, - # 999 - # ] - # }, - # "1_week": { - # "count": 169, - # "factor": 0.001001001001001001, - # "first": "2015-03-23 07:30:00", - # "interval": 3600, - # "last": "2015-03-30 07:30:00", - # "values": [ ...] - # }, - # "1_year": { - # "count": 177, - # "factor": 0.001001001001001001, - # "first": "2014-04-11 00:00:00", - # "interval": 172800, - # "last": "2015-03-29 00:00:00", - # "values": [ ...] - # }, - # "3_months": { - # "count": 185, - # "factor": 0.001001001001001001, - # "first": "2014-12-28 06:00:00", - # "interval": 43200, - # "last": "2015-03-30 06:00:00", - # "values": [ ...] - # } - # }, - # extract exactly one piece of data per time interval, - # using smaller intervals where available. - # - # returns list of (age, length, value) dictionaries. - - generic_history = [] - - periods = history.keys() - periods.sort(key = lambda x: history[x]['interval']) - now = datetime.datetime.utcnow() - newest = now - for p in periods: - h = history[p] - interval = datetime.timedelta(seconds = h['interval']) - this_ts = parse_ts(h['last']) - - if (len(h['values']) != h['count']): - logging.warning('Inconsistent value count in %s document for %s' - %(p, which)) - for v in reversed(h['values']): - if (this_ts <= newest): - agt1 = now - this_ts - agt2 = interval - agetmp1 = (agt1.microseconds + (agt1.seconds + agt1.days * 24 * 3600) - * 10**6) / 10**6 - agetmp2 = (agt2.microseconds + (agt2.seconds + agt2.days * 24 * 3600) - * 10**6) / 10**6 - generic_history.append( - { 'age': agetmp1, - 'length': agetmp2, - 'value': v - }) - newest = this_ts - this_ts -= interval - - if (this_ts + interval != parse_ts(h['first'])): - logging.warning('Inconsistent time information in %s document for %s' - %(p, which)) - - #print json.dumps(generic_history, sort_keys=True, - # indent=4, separators=(',', ': ')) - return generic_history - - @staticmethod - def _avg_generic_history(generic_history): - a = [] - for i in generic_history: - if i['age'] > (ADDRESS_AND_PORT_STABLE_DAYS * 24 * 3600): - continue - if (i['length'] is not None - and i['age'] is not None - and i['value'] is not None): - w = i['length'] * math.pow(AGE_ALPHA, i['age']/(3600*24)) - a.append( (i['value'] * w, w) ) - - sv = math.fsum(map(lambda x: x[0], a)) - sw = math.fsum(map(lambda x: x[1], a)) - - if sw == 0.0: - svw = 0.0 - else: - svw = sv/sw - return svw - - def _add_generic_history(self, history): - periods = r['read_history'].keys() - periods.sort(key = lambda x: r['read_history'][x]['interval'] ) - - print periods - - def add_running_history(self, history): - pass - - def add_uptime(self, uptime): - logging.debug('Adding uptime %s.'%(self._fpr,)) - - # flags we care about: Running, V2Dir, Guard - if not 'flags' in uptime: - logging.debug('No flags in document for %s.'%(self._fpr,)) - return - - for f in ['Running', 'Guard', 'V2Dir']: - if not f in uptime['flags']: - logging.debug('No %s in flags for %s.'%(f, self._fpr,)) - return - - running = self._extract_generic_history(uptime['flags']['Running'], - '%s-Running'%(self._fpr)) - guard = self._extract_generic_history(uptime['flags']['Guard'], - '%s-Guard'%(self._fpr)) - v2dir = self._extract_generic_history(uptime['flags']['V2Dir'], - '%s-V2Dir'%(self._fpr)) - if 'BadExit' in uptime['flags']: - badexit = self._extract_generic_history(uptime['flags']['BadExit'], - '%s-BadExit'%(self._fpr)) - - self._running = self._avg_generic_history(running) / ONIONOO_SCALE_ONE - self._guard = self._avg_generic_history(guard) / ONIONOO_SCALE_ONE - self._v2dir = self._avg_generic_history(v2dir) / ONIONOO_SCALE_ONE - self._badexit = None - if 'BadExit' in uptime['flags']: - self._badexit = self._avg_generic_history(badexit) / ONIONOO_SCALE_ONE - - def is_candidate(self): - must_be_running_now = (PERFORM_IPV4_DIRPORT_CHECKS - or PERFORM_IPV6_DIRPORT_CHECKS) - if (must_be_running_now and not self.is_running()): - logging.info('%s not a candidate: not running now, unable to check ' + - 'DirPort consensus download', self._fpr) - return False - if (self._data['last_changed_address_or_port'] > - self.CUTOFF_ADDRESS_AND_PORT_STABLE): - logging.info('%s not a candidate: changed address/port recently (%s)', - self._fpr, self._data['last_changed_address_or_port']) - return False - if self._running < CUTOFF_RUNNING: - logging.info('%s not a candidate: running avg too low (%lf)', - self._fpr, self._running) - return False - if self._v2dir < CUTOFF_V2DIR: - logging.info('%s not a candidate: v2dir avg too low (%lf)', - self._fpr, self._v2dir) - return False - if self._badexit is not None and self._badexit > PERMITTED_BADEXIT: - logging.info('%s not a candidate: badexit avg too high (%lf)', - self._fpr, self._badexit) - return False - # if the relay doesn't report a version, also exclude the relay - if (not self._data.has_key('recommended_version') - or not self._data['recommended_version']): - logging.info('%s not a candidate: version not recommended', self._fpr) - return False - if self._guard < CUTOFF_GUARD: - logging.info('%s not a candidate: guard avg too low (%lf)', - self._fpr, self._guard) - return False - if (not self._data.has_key('consensus_weight') - or self._data['consensus_weight'] < 1): - logging.info('%s not a candidate: consensus weight invalid', self._fpr) - return False - return True - - def is_in_whitelist(self, relaylist): - """ A fallback matches if each key in the whitelist line matches: - ipv4 - dirport - orport - id - ipv6 address and port (if present) - If the fallback has an ipv6 key, the whitelist line must also have - it, and vice versa, otherwise they don't match. """ - ipv6 = None - if self.has_ipv6(): - ipv6 = '%s:%d'%(self.ipv6addr, self.ipv6orport) - for entry in relaylist: - if entry['id'] != self._fpr: - # can't log here unless we match an IP and port, because every relay's - # fingerprint is compared to every entry's fingerprint - if entry['ipv4'] == self.dirip and int(entry['orport']) == self.orport: - logging.warning('%s excluded: has OR %s:%d changed fingerprint to ' + - '%s?', entry['id'], self.dirip, self.orport, - self._fpr) - if self.has_ipv6() and entry.has_key('ipv6') and entry['ipv6'] == ipv6: - logging.warning('%s excluded: has OR %s changed fingerprint to ' + - '%s?', entry['id'], ipv6, self._fpr) - continue - if entry['ipv4'] != self.dirip: - logging.warning('%s excluded: has it changed IPv4 from %s to %s?', - self._fpr, entry['ipv4'], self.dirip) - continue - if int(entry['dirport']) != self.dirport: - logging.warning('%s excluded: has it changed DirPort from %s:%d to ' + - '%s:%d?', self._fpr, self.dirip, int(entry['dirport']), - self.dirip, self.dirport) - continue - if int(entry['orport']) != self.orport: - logging.warning('%s excluded: has it changed ORPort from %s:%d to ' + - '%s:%d?', self._fpr, self.dirip, int(entry['orport']), - self.dirip, self.orport) - continue - if entry.has_key('ipv6') and self.has_ipv6(): - # if both entry and fallback have an ipv6 address, compare them - if entry['ipv6'] != ipv6: - logging.warning('%s excluded: has it changed IPv6 ORPort from %s ' + - 'to %s?', self._fpr, entry['ipv6'], ipv6) - continue - # if the fallback has an IPv6 address but the whitelist entry - # doesn't, or vice versa, the whitelist entry doesn't match - elif entry.has_key('ipv6') and not self.has_ipv6(): - logging.warning('%s excluded: has it lost its former IPv6 address %s?', - self._fpr, entry['ipv6']) - continue - elif not entry.has_key('ipv6') and self.has_ipv6(): - logging.warning('%s excluded: has it gained an IPv6 address %s?', - self._fpr, ipv6) - continue - return True - return False - - def is_in_blacklist(self, relaylist): - """ A fallback matches a blacklist line if a sufficiently specific group - of attributes matches: - ipv4 & dirport - ipv4 & orport - id - ipv6 & dirport - ipv6 & ipv6 orport - If the fallback and the blacklist line both have an ipv6 key, - their values will be compared, otherwise, they will be ignored. - If there is no dirport and no orport, the entry matches all relays on - that ip. """ - for entry in relaylist: - for key in entry: - value = entry[key] - if key == 'id' and value == self._fpr: - logging.info('%s is in the blacklist: fingerprint matches', - self._fpr) - return True - if key == 'ipv4' and value == self.dirip: - # if the dirport is present, check it too - if entry.has_key('dirport'): - if int(entry['dirport']) == self.dirport: - logging.info('%s is in the blacklist: IPv4 (%s) and ' + - 'DirPort (%d) match', self._fpr, self.dirip, - self.dirport) - return True - # if the orport is present, check it too - elif entry.has_key('orport'): - if int(entry['orport']) == self.orport: - logging.info('%s is in the blacklist: IPv4 (%s) and ' + - 'ORPort (%d) match', self._fpr, self.dirip, - self.orport) - return True - else: - logging.info('%s is in the blacklist: IPv4 (%s) matches, and ' + - 'entry has no DirPort or ORPort', self._fpr, - self.dirip) - return True - ipv6 = None - if self.has_ipv6(): - ipv6 = '%s:%d'%(self.ipv6addr, self.ipv6orport) - if (key == 'ipv6' and self.has_ipv6()): - # if both entry and fallback have an ipv6 address, compare them, - # otherwise, disregard ipv6 addresses - if value == ipv6: - # if the dirport is present, check it too - if entry.has_key('dirport'): - if int(entry['dirport']) == self.dirport: - logging.info('%s is in the blacklist: IPv6 (%s) and ' + - 'DirPort (%d) match', self._fpr, ipv6, - self.dirport) - return True - # we've already checked the ORPort, it's part of entry['ipv6'] - else: - logging.info('%s is in the blacklist: IPv6 (%s) matches, and' + - 'entry has no DirPort', self._fpr, ipv6) - return True - elif (key == 'ipv6' or self.has_ipv6()): - # only log if the fingerprint matches but the IPv6 doesn't - if entry.has_key('id') and entry['id'] == self._fpr: - logging.info('%s skipping IPv6 blacklist comparison: relay ' + - 'has%s IPv6%s, but entry has%s IPv6%s', self._fpr, - '' if self.has_ipv6() else ' no', - (' (' + ipv6 + ')') if self.has_ipv6() else '', - '' if key == 'ipv6' else ' no', - (' (' + value + ')') if key == 'ipv6' else '') - logging.warning('Has %s %s IPv6 address %s?', self._fpr, - 'gained an' if self.has_ipv6() else 'lost its former', - ipv6 if self.has_ipv6() else value) - return False - - def cw_to_bw_factor(self): - # any relays with a missing or zero consensus weight are not candidates - # any relays with a missing advertised bandwidth have it set to zero - return self._data['advertised_bandwidth'] / self._data['consensus_weight'] - - # since advertised_bandwidth is reported by the relay, it can be gamed - # to avoid this, use the median consensus weight to bandwidth factor to - # estimate this relay's measured bandwidth, and make that the upper limit - def measured_bandwidth(self, median_cw_to_bw_factor): - cw_to_bw= median_cw_to_bw_factor - # Reduce exit bandwidth to make sure we're not overloading them - if self.is_exit(): - cw_to_bw *= EXIT_BANDWIDTH_FRACTION - measured_bandwidth = self._data['consensus_weight'] * cw_to_bw - if self._data['advertised_bandwidth'] != 0: - # limit advertised bandwidth (if available) to measured bandwidth - return min(measured_bandwidth, self._data['advertised_bandwidth']) - else: - return measured_bandwidth - - def set_measured_bandwidth(self, median_cw_to_bw_factor): - self._data['measured_bandwidth'] = self.measured_bandwidth( - median_cw_to_bw_factor) - - def is_exit(self): - return 'Exit' in self._data['flags'] - - def is_guard(self): - return 'Guard' in self._data['flags'] - - def is_running(self): - return 'Running' in self._data['flags'] - - # does this fallback have an IPv6 address and orport? - def has_ipv6(self): - return self.ipv6addr is not None and self.ipv6orport is not None - - # strip leading and trailing brackets from an IPv6 address - # safe to use on non-bracketed IPv6 and on IPv4 addresses - # also convert to unicode, and make None appear as '' - @staticmethod - def strip_ipv6_brackets(ip): - if ip is None: - return unicode('') - if len(ip) < 2: - return unicode(ip) - if ip[0] == '[' and ip[-1] == ']': - return unicode(ip[1:-1]) - return unicode(ip) - - # are ip_a and ip_b in the same netblock? - # mask_bits is the size of the netblock - # takes both IPv4 and IPv6 addresses - # the versions of ip_a and ip_b must be the same - # the mask must be valid for the IP version - @staticmethod - def netblocks_equal(ip_a, ip_b, mask_bits): - if ip_a is None or ip_b is None: - return False - ip_a = Candidate.strip_ipv6_brackets(ip_a) - ip_b = Candidate.strip_ipv6_brackets(ip_b) - a = ipaddress.ip_address(ip_a) - b = ipaddress.ip_address(ip_b) - if a.version != b.version: - raise Exception('Mismatching IP versions in %s and %s'%(ip_a, ip_b)) - if mask_bits > a.max_prefixlen: - logging.error('Bad IP mask %d for %s and %s'%(mask_bits, ip_a, ip_b)) - mask_bits = a.max_prefixlen - if mask_bits < 0: - logging.error('Bad IP mask %d for %s and %s'%(mask_bits, ip_a, ip_b)) - mask_bits = 0 - a_net = ipaddress.ip_network('%s/%d'%(ip_a, mask_bits), strict=False) - return b in a_net - - # is this fallback's IPv4 address (dirip) in the same netblock as other's - # IPv4 address? - # mask_bits is the size of the netblock - def ipv4_netblocks_equal(self, other, mask_bits): - return Candidate.netblocks_equal(self.dirip, other.dirip, mask_bits) - - # is this fallback's IPv6 address (ipv6addr) in the same netblock as - # other's IPv6 address? - # Returns False if either fallback has no IPv6 address - # mask_bits is the size of the netblock - def ipv6_netblocks_equal(self, other, mask_bits): - if not self.has_ipv6() or not other.has_ipv6(): - return False - return Candidate.netblocks_equal(self.ipv6addr, other.ipv6addr, mask_bits) - - # is this fallback's IPv4 DirPort the same as other's IPv4 DirPort? - def dirport_equal(self, other): - return self.dirport == other.dirport - - # is this fallback's IPv4 ORPort the same as other's IPv4 ORPort? - def ipv4_orport_equal(self, other): - return self.orport == other.orport - - # is this fallback's IPv6 ORPort the same as other's IPv6 ORPort? - # Returns False if either fallback has no IPv6 address - def ipv6_orport_equal(self, other): - if not self.has_ipv6() or not other.has_ipv6(): - return False - return self.ipv6orport == other.ipv6orport - - # does this fallback have the same DirPort, IPv4 ORPort, or - # IPv6 ORPort as other? - # Ignores IPv6 ORPort if either fallback has no IPv6 address - def port_equal(self, other): - return (self.dirport_equal(other) or self.ipv4_orport_equal(other) - or self.ipv6_orport_equal(other)) - - # return a list containing IPv4 ORPort, DirPort, and IPv6 ORPort (if present) - def port_list(self): - ports = [self.dirport, self.orport] - if self.has_ipv6() and not self.ipv6orport in ports: - ports.append(self.ipv6orport) - return ports - - # does this fallback share a port with other, regardless of whether the - # port types match? - # For example, if self's IPv4 ORPort is 80 and other's DirPort is 80, - # return True - def port_shared(self, other): - for p in self.port_list(): - if p in other.port_list(): - return True - return False - - # report how long it takes to download a consensus from dirip:dirport - @staticmethod - def fallback_consensus_download_speed(dirip, dirport, nickname, max_time): - download_failed = False - downloader = DescriptorDownloader() - start = datetime.datetime.utcnow() - # some directory mirrors respond to requests in ways that hang python - # sockets, which is why we log this line here - logging.info('Initiating consensus download from %s (%s:%d).', nickname, - dirip, dirport) - # there appears to be about 1 second of overhead when comparing stem's - # internal trace time and the elapsed time calculated here - TIMEOUT_SLOP = 1.0 - try: - downloader.get_consensus(endpoints = [(dirip, dirport)], - timeout = (max_time + TIMEOUT_SLOP), - validate = True, - retries = 0, - fall_back_to_authority = False).run() - except Exception, stem_error: - logging.info('Unable to retrieve a consensus from %s: %s', nickname, - stem_error) - status = 'error: "%s"' % (stem_error) - level = logging.WARNING - download_failed = True - elapsed = (datetime.datetime.utcnow() - start).total_seconds() - if elapsed > max_time: - status = 'too slow' - level = logging.WARNING - download_failed = True - else: - status = 'ok' - level = logging.DEBUG - logging.log(level, 'Consensus download: %0.1fs %s from %s (%s:%d), ' + - 'max download time %0.1fs.', elapsed, status, nickname, - dirip, dirport, max_time) - return download_failed - - # does this fallback download the consensus fast enough? - def check_fallback_download_consensus(self): - # include the relay if we're not doing a check, or we can't check (IPv6) - ipv4_failed = False - ipv6_failed = False - if PERFORM_IPV4_DIRPORT_CHECKS: - ipv4_failed = Candidate.fallback_consensus_download_speed(self.dirip, - self.dirport, - self._data['nickname'], - CONSENSUS_DOWNLOAD_SPEED_MAX) - if self.has_ipv6() and PERFORM_IPV6_DIRPORT_CHECKS: - # Clients assume the IPv6 DirPort is the same as the IPv4 DirPort - ipv6_failed = Candidate.fallback_consensus_download_speed(self.ipv6addr, - self.dirport, - self._data['nickname'], - CONSENSUS_DOWNLOAD_SPEED_MAX) - return ((not ipv4_failed) and (not ipv6_failed)) - - # if this fallback has not passed a download check, try it again, - # and record the result, available in get_fallback_download_consensus - def try_fallback_download_consensus(self): - if not self.get_fallback_download_consensus(): - self._data['download_check'] = self.check_fallback_download_consensus() - - # did this fallback pass the download check? - def get_fallback_download_consensus(self): - # if we're not performing checks, return True - if not PERFORM_IPV4_DIRPORT_CHECKS and not PERFORM_IPV6_DIRPORT_CHECKS: - return True - # if we are performing checks, but haven't done one, return False - if not self._data.has_key('download_check'): - return False - return self._data['download_check'] - - # output an optional header comment and info for this fallback - # try_fallback_download_consensus before calling this - def fallbackdir_line(self, fallbacks, prefilter_fallbacks): - s = '' - if OUTPUT_COMMENTS: - s += self.fallbackdir_comment(fallbacks, prefilter_fallbacks) - # if the download speed is ok, output a C string - # if it's not, but we OUTPUT_COMMENTS, output a commented-out C string - if self.get_fallback_download_consensus() or OUTPUT_COMMENTS: - s += self.fallbackdir_info(self.get_fallback_download_consensus()) - return s - - # output a header comment for this fallback - def fallbackdir_comment(self, fallbacks, prefilter_fallbacks): - # /* - # nickname - # flags - # [contact] - # [identical contact counts] - # */ - # Multiline C comment - s = '/*' - s += '\n' - s += cleanse_c_multiline_comment(self._data['nickname']) - s += '\n' - s += 'Flags: ' - s += cleanse_c_multiline_comment(' '.join(sorted(self._data['flags']))) - s += '\n' - if self._data['contact'] is not None: - s += cleanse_c_multiline_comment(self._data['contact']) - if CONTACT_COUNT or CONTACT_BLACKLIST_COUNT: - fallback_count = len([f for f in fallbacks - if f._data['contact'] == self._data['contact']]) - if fallback_count > 1: - s += '\n' - s += '%d identical contacts listed' % (fallback_count) - if CONTACT_BLACKLIST_COUNT: - prefilter_count = len([f for f in prefilter_fallbacks - if f._data['contact'] == self._data['contact']]) - filter_count = prefilter_count - fallback_count - if filter_count > 0: - if fallback_count > 1: - s += ' ' - else: - s += '\n' - s += '%d blacklisted' % (filter_count) - s += '\n' - s += '*/' - s += '\n' - - # output the fallback info C string for this fallback - # this is the text that would go after FallbackDir in a torrc - # if this relay failed the download test and we OUTPUT_COMMENTS, - # comment-out the returned string - def fallbackdir_info(self, dl_speed_ok): - # "address:dirport orport=port id=fingerprint" - # "[ipv6=addr:orport]" - # "weight=FALLBACK_OUTPUT_WEIGHT", - # - # Do we want a C string, or a commented-out string? - c_string = dl_speed_ok - comment_string = not dl_speed_ok and OUTPUT_COMMENTS - # If we don't want either kind of string, bail - if not c_string and not comment_string: - return '' - s = '' - # Comment out the fallback directory entry if it's too slow - # See the debug output for which address and port is failing - if comment_string: - s += '/* Consensus download failed or was too slow:\n' - # Multi-Line C string with trailing comma (part of a string list) - # This makes it easier to diff the file, and remove IPv6 lines using grep - # Integers don't need escaping - s += '"%s orport=%d id=%s"'%( - cleanse_c_string(self._data['dir_address']), - self.orport, - cleanse_c_string(self._fpr)) - s += '\n' - if self.has_ipv6(): - s += '" ipv6=%s:%d"'%(cleanse_c_string(self.ipv6addr), self.ipv6orport) - s += '\n' - s += '" weight=%d",'%(FALLBACK_OUTPUT_WEIGHT) - if comment_string: - s += '\n' - s += '*/' - return s - -## Fallback Candidate List Class - -class CandidateList(dict): - def __init__(self): - pass - - def _add_relay(self, details): - if not 'dir_address' in details: return - c = Candidate(details) - self[ c.get_fingerprint() ] = c - - def _add_uptime(self, uptime): - try: - fpr = uptime['fingerprint'] - except KeyError: - raise Exception("Document has no fingerprint field.") - - try: - c = self[fpr] - except KeyError: - logging.debug('Got unknown relay %s in uptime document.'%(fpr,)) - return - - c.add_uptime(uptime) - - def _add_details(self): - logging.debug('Loading details document.') - d = fetch('details', - fields=('fingerprint,nickname,contact,last_changed_address_or_port,' + - 'consensus_weight,advertised_bandwidth,or_addresses,' + - 'dir_address,recommended_version,flags,effective_family')) - logging.debug('Loading details document done.') - - if not 'relays' in d: raise Exception("No relays found in document.") - - for r in d['relays']: self._add_relay(r) - - def _add_uptimes(self): - logging.debug('Loading uptime document.') - d = fetch('uptime') - logging.debug('Loading uptime document done.') - - if not 'relays' in d: raise Exception("No relays found in document.") - for r in d['relays']: self._add_uptime(r) - - def add_relays(self): - self._add_details() - self._add_uptimes() - - def count_guards(self): - guard_count = 0 - for fpr in self.keys(): - if self[fpr].is_guard(): - guard_count += 1 - return guard_count - - # Find fallbacks that fit the uptime, stability, and flags criteria, - # and make an array of them in self.fallbacks - def compute_fallbacks(self): - self.fallbacks = map(lambda x: self[x], - filter(lambda x: self[x].is_candidate(), - self.keys())) - - # sort fallbacks by their consensus weight to advertised bandwidth factor, - # lowest to highest - # used to find the median cw_to_bw_factor() - def sort_fallbacks_by_cw_to_bw_factor(self): - self.fallbacks.sort(key=lambda f: f.cw_to_bw_factor()) - - # sort fallbacks by their measured bandwidth, highest to lowest - # calculate_measured_bandwidth before calling this - # this is useful for reviewing candidates in priority order - def sort_fallbacks_by_measured_bandwidth(self): - self.fallbacks.sort(key=lambda f: f._data['measured_bandwidth'], - reverse=True) - - # sort fallbacks by their fingerprint, lowest to highest - # this is useful for stable diffs of fallback lists - def sort_fallbacks_by_fingerprint(self): - self.fallbacks.sort(key=lambda f: f._fpr) - - @staticmethod - def load_relaylist(file_name): - """ Read each line in the file, and parse it like a FallbackDir line: - an IPv4 address and optional port: - <IPv4 address>:<port> - which are parsed into dictionary entries: - ipv4=<IPv4 address> - dirport=<port> - followed by a series of key=value entries: - orport=<port> - id=<fingerprint> - ipv6=<IPv6 address>:<IPv6 orport> - each line's key/value pairs are placed in a dictonary, - (of string -> string key/value pairs), - and these dictionaries are placed in an array. - comments start with # and are ignored """ - relaylist = [] - file_data = read_from_file(file_name, MAX_LIST_FILE_SIZE) - if file_data is None: - return relaylist - for line in file_data.split('\n'): - relay_entry = {} - # ignore comments - line_comment_split = line.split('#') - line = line_comment_split[0] - # cleanup whitespace - line = cleanse_whitespace(line) - line = line.strip() - if len(line) == 0: - continue - for item in line.split(' '): - item = item.strip() - if len(item) == 0: - continue - key_value_split = item.split('=') - kvl = len(key_value_split) - if kvl < 1 or kvl > 2: - print '#error Bad %s item: %s, format is key=value.'%( - file_name, item) - if kvl == 1: - # assume that entries without a key are the ipv4 address, - # perhaps with a dirport - ipv4_maybe_dirport = key_value_split[0] - ipv4_maybe_dirport_split = ipv4_maybe_dirport.split(':') - dirl = len(ipv4_maybe_dirport_split) - if dirl < 1 or dirl > 2: - print '#error Bad %s IPv4 item: %s, format is ipv4:port.'%( - file_name, item) - if dirl >= 1: - relay_entry['ipv4'] = ipv4_maybe_dirport_split[0] - if dirl == 2: - relay_entry['dirport'] = ipv4_maybe_dirport_split[1] - elif kvl == 2: - relay_entry[key_value_split[0]] = key_value_split[1] - relaylist.append(relay_entry) - return relaylist - - # apply the fallback whitelist and blacklist - def apply_filter_lists(self): - excluded_count = 0 - logging.debug('Applying whitelist and blacklist.') - # parse the whitelist and blacklist - whitelist = self.load_relaylist(WHITELIST_FILE_NAME) - blacklist = self.load_relaylist(BLACKLIST_FILE_NAME) - filtered_fallbacks = [] - for f in self.fallbacks: - in_whitelist = f.is_in_whitelist(whitelist) - in_blacklist = f.is_in_blacklist(blacklist) - if in_whitelist and in_blacklist: - if BLACKLIST_EXCLUDES_WHITELIST_ENTRIES: - # exclude - excluded_count += 1 - logging.warning('Excluding %s: in both blacklist and whitelist.', - f._fpr) - else: - # include - filtered_fallbacks.append(f) - elif in_whitelist: - # include - filtered_fallbacks.append(f) - elif in_blacklist: - # exclude - excluded_count += 1 - logging.info('Excluding %s: in blacklist.', f._fpr) - else: - if INCLUDE_UNLISTED_ENTRIES: - # include - filtered_fallbacks.append(f) - else: - # exclude - excluded_count += 1 - logging.info('Excluding %s: in neither blacklist nor whitelist.', - f._fpr) - self.fallbacks = filtered_fallbacks - return excluded_count - - @staticmethod - def summarise_filters(initial_count, excluded_count): - return '/* Whitelist & blacklist excluded %d of %d candidates. */'%( - excluded_count, initial_count) - - # calculate each fallback's measured bandwidth based on the median - # consensus weight to advertised bandwdith ratio - def calculate_measured_bandwidth(self): - self.sort_fallbacks_by_cw_to_bw_factor() - median_fallback = self.fallback_median(True) - if median_fallback is not None: - median_cw_to_bw_factor = median_fallback.cw_to_bw_factor() - else: - # this will never be used, because there are no fallbacks - median_cw_to_bw_factor = None - for f in self.fallbacks: - f.set_measured_bandwidth(median_cw_to_bw_factor) - - # remove relays with low measured bandwidth from the fallback list - # calculate_measured_bandwidth for each relay before calling this - def remove_low_bandwidth_relays(self): - if MIN_BANDWIDTH is None: - return - above_min_bw_fallbacks = [] - for f in self.fallbacks: - if f._data['measured_bandwidth'] >= MIN_BANDWIDTH: - above_min_bw_fallbacks.append(f) - else: - # the bandwidth we log here is limited by the relay's consensus weight - # as well as its adverttised bandwidth. See set_measured_bandwidth - # for details - logging.info('%s not a candidate: bandwidth %.1fMB/s too low, must ' + - 'be at least %.1fMB/s', f._fpr, - f._data['measured_bandwidth']/(1024.0*1024.0), - MIN_BANDWIDTH/(1024.0*1024.0)) - self.fallbacks = above_min_bw_fallbacks - - # the minimum fallback in the list - # call one of the sort_fallbacks_* functions before calling this - def fallback_min(self): - if len(self.fallbacks) > 0: - return self.fallbacks[-1] - else: - return None - - # the median fallback in the list - # call one of the sort_fallbacks_* functions before calling this - def fallback_median(self, require_advertised_bandwidth): - # use the low-median when there are an evan number of fallbacks, - # for consistency with the bandwidth authorities - if len(self.fallbacks) > 0: - median_position = (len(self.fallbacks) - 1) / 2 - if not require_advertised_bandwidth: - return self.fallbacks[median_position] - # if we need advertised_bandwidth but this relay doesn't have it, - # move to a fallback with greater consensus weight until we find one - while not self.fallbacks[median_position]._data['advertised_bandwidth']: - median_position += 1 - if median_position >= len(self.fallbacks): - return None - return self.fallbacks[median_position] - else: - return None - - # the maximum fallback in the list - # call one of the sort_fallbacks_* functions before calling this - def fallback_max(self): - if len(self.fallbacks) > 0: - return self.fallbacks[0] - else: - return None - - # does exclusion_list contain attribute? - # if so, return False - # if not, return True - # if attribute is None or the empty string, always return True - @staticmethod - def allow(attribute, exclusion_list): - if attribute is None or attribute == '': - return True - elif attribute in exclusion_list: - return False - else: - return True - - # make sure there is only one fallback per IPv4 address, and per IPv6 address - # there is only one IPv4 address on each fallback: the IPv4 DirPort address - # (we choose the IPv4 ORPort which is on the same IPv4 as the DirPort) - # there is at most one IPv6 address on each fallback: the IPv6 ORPort address - # we try to match the IPv4 ORPort, but will use any IPv6 address if needed - # (clients assume the IPv6 DirPort is the same as the IPv4 DirPort, but - # typically only use the IPv6 ORPort) - # if there is no IPv6 address, only the IPv4 address is checked - # return the number of candidates we excluded - def limit_fallbacks_same_ip(self): - ip_limit_fallbacks = [] - ip_list = [] - for f in self.fallbacks: - if (CandidateList.allow(f.dirip, ip_list) - and CandidateList.allow(f.ipv6addr, ip_list)): - ip_limit_fallbacks.append(f) - ip_list.append(f.dirip) - if f.has_ipv6(): - ip_list.append(f.ipv6addr) - elif not CandidateList.allow(f.dirip, ip_list): - logging.info('Eliminated %s: already have fallback on IPv4 %s'%( - f._fpr, f.dirip)) - elif f.has_ipv6() and not CandidateList.allow(f.ipv6addr, ip_list): - logging.info('Eliminated %s: already have fallback on IPv6 %s'%( - f._fpr, f.ipv6addr)) - original_count = len(self.fallbacks) - self.fallbacks = ip_limit_fallbacks - return original_count - len(self.fallbacks) - - # make sure there is only one fallback per ContactInfo - # if there is no ContactInfo, allow the fallback - # this check can be gamed by providing no ContactInfo, or by setting the - # ContactInfo to match another fallback - # However, given the likelihood that relays with the same ContactInfo will - # go down at similar times, its usefulness outweighs the risk - def limit_fallbacks_same_contact(self): - contact_limit_fallbacks = [] - contact_list = [] - for f in self.fallbacks: - if CandidateList.allow(f._data['contact'], contact_list): - contact_limit_fallbacks.append(f) - contact_list.append(f._data['contact']) - else: - logging.info(('Eliminated %s: already have fallback on ' + - 'ContactInfo %s')%(f._fpr, f._data['contact'])) - original_count = len(self.fallbacks) - self.fallbacks = contact_limit_fallbacks - return original_count - len(self.fallbacks) - - # make sure there is only one fallback per effective family - # if there is no family, allow the fallback - # this check can't be gamed, because we use effective family, which ensures - # mutual family declarations - # if any indirect families exist, the result depends on the order in which - # fallbacks are sorted in the list - def limit_fallbacks_same_family(self): - family_limit_fallbacks = [] - fingerprint_list = [] - for f in self.fallbacks: - if CandidateList.allow(f._fpr, fingerprint_list): - family_limit_fallbacks.append(f) - fingerprint_list.append(f._fpr) - fingerprint_list.extend(f._data['effective_family']) - else: - # technically, we already have a fallback with this fallback in its - # effective family - logging.info('Eliminated %s: already have fallback in effective ' + - 'family'%(f._fpr)) - original_count = len(self.fallbacks) - self.fallbacks = family_limit_fallbacks - return original_count - len(self.fallbacks) - - # try a download check on each fallback candidate in order - # stop after max_count successful downloads - # but don't remove any candidates from the array - def try_download_consensus_checks(self, max_count): - dl_ok_count = 0 - for f in self.fallbacks: - f.try_fallback_download_consensus() - if f.get_fallback_download_consensus(): - # this fallback downloaded a consensus ok - dl_ok_count += 1 - if dl_ok_count >= max_count: - # we have enough fallbacks - return - - # put max_count successful candidates in the fallbacks array: - # - perform download checks on each fallback candidate - # - retry failed candidates if CONSENSUS_DOWNLOAD_RETRY is set - # - eliminate failed candidates - # - if there are more than max_count candidates, eliminate lowest bandwidth - # - if there are fewer than max_count candidates, leave only successful - # Return the number of fallbacks that failed the consensus check - def perform_download_consensus_checks(self, max_count): - self.sort_fallbacks_by_measured_bandwidth() - self.try_download_consensus_checks(max_count) - if CONSENSUS_DOWNLOAD_RETRY: - # try unsuccessful candidates again - # we could end up with more than max_count successful candidates here - self.try_download_consensus_checks(max_count) - # now we have at least max_count successful candidates, - # or we've tried them all - original_count = len(self.fallbacks) - self.fallbacks = filter(lambda x: x.get_fallback_download_consensus(), - self.fallbacks) - # some of these failed the check, others skipped the check, - # if we already had enough successful downloads - failed_count = original_count - len(self.fallbacks) - self.fallbacks = self.fallbacks[:max_count] - return failed_count - - # return a string that describes a/b as a percentage - @staticmethod - def describe_percentage(a, b): - if b != 0: - return '%d/%d = %.0f%%'%(a, b, (a*100.0)/b) - else: - # technically, 0/0 is undefined, but 0.0% is a sensible result - return '%d/%d = %.0f%%'%(a, b, 0.0) - - # return a dictionary of lists of fallbacks by IPv4 netblock - # the dictionary is keyed by the fingerprint of an arbitrary fallback - # in each netblock - # mask_bits is the size of the netblock - def fallbacks_by_ipv4_netblock(self, mask_bits): - netblocks = {} - for f in self.fallbacks: - found_netblock = False - for b in netblocks.keys(): - # we found an existing netblock containing this fallback - if f.ipv4_netblocks_equal(self[b], mask_bits): - # add it to the list - netblocks[b].append(f) - found_netblock = True - break - # make a new netblock based on this fallback's fingerprint - if not found_netblock: - netblocks[f._fpr] = [f] - return netblocks - - # return a dictionary of lists of fallbacks by IPv6 netblock - # where mask_bits is the size of the netblock - def fallbacks_by_ipv6_netblock(self, mask_bits): - netblocks = {} - for f in self.fallbacks: - # skip fallbacks without IPv6 addresses - if not f.has_ipv6(): - continue - found_netblock = False - for b in netblocks.keys(): - # we found an existing netblock containing this fallback - if f.ipv6_netblocks_equal(self[b], mask_bits): - # add it to the list - netblocks[b].append(f) - found_netblock = True - break - # make a new netblock based on this fallback's fingerprint - if not found_netblock: - netblocks[f._fpr] = [f] - return netblocks - - # log a message about the proportion of fallbacks in each IPv4 netblock, - # where mask_bits is the size of the netblock - def describe_fallback_ipv4_netblock_mask(self, mask_bits): - fallback_count = len(self.fallbacks) - shared_netblock_fallback_count = 0 - most_frequent_netblock = None - netblocks = self.fallbacks_by_ipv4_netblock(mask_bits) - for b in netblocks.keys(): - if len(netblocks[b]) > 1: - # how many fallbacks are in a netblock with other fallbacks? - shared_netblock_fallback_count += len(netblocks[b]) - # what's the netblock with the most fallbacks? - if (most_frequent_netblock is None - or len(netblocks[b]) > len(netblocks[most_frequent_netblock])): - most_frequent_netblock = b - logging.debug('Fallback IPv4 addresses in the same /%d:'%(mask_bits)) - for f in netblocks[b]: - logging.debug('%s - %s', f.dirip, f._fpr) - if most_frequent_netblock is not None: - logging.warning('There are %s fallbacks in the IPv4 /%d containing %s'%( - CandidateList.describe_percentage( - len(netblocks[most_frequent_netblock]), - fallback_count), - mask_bits, - self[most_frequent_netblock].dirip)) - if shared_netblock_fallback_count > 0: - logging.warning(('%s of fallbacks are in an IPv4 /%d with other ' + - 'fallbacks')%(CandidateList.describe_percentage( - shared_netblock_fallback_count, - fallback_count), - mask_bits)) - - # log a message about the proportion of fallbacks in each IPv6 netblock, - # where mask_bits is the size of the netblock - def describe_fallback_ipv6_netblock_mask(self, mask_bits): - fallback_count = len(self.fallbacks_with_ipv6()) - shared_netblock_fallback_count = 0 - most_frequent_netblock = None - netblocks = self.fallbacks_by_ipv6_netblock(mask_bits) - for b in netblocks.keys(): - if len(netblocks[b]) > 1: - # how many fallbacks are in a netblock with other fallbacks? - shared_netblock_fallback_count += len(netblocks[b]) - # what's the netblock with the most fallbacks? - if (most_frequent_netblock is None - or len(netblocks[b]) > len(netblocks[most_frequent_netblock])): - most_frequent_netblock = b - logging.debug('Fallback IPv6 addresses in the same /%d:'%(mask_bits)) - for f in netblocks[b]: - logging.debug('%s - %s', f.ipv6addr, f._fpr) - if most_frequent_netblock is not None: - logging.warning('There are %s fallbacks in the IPv6 /%d containing %s'%( - CandidateList.describe_percentage( - len(netblocks[most_frequent_netblock]), - fallback_count), - mask_bits, - self[most_frequent_netblock].ipv6addr)) - if shared_netblock_fallback_count > 0: - logging.warning(('%s of fallbacks are in an IPv6 /%d with other ' + - 'fallbacks')%(CandidateList.describe_percentage( - shared_netblock_fallback_count, - fallback_count), - mask_bits)) - - # log a message about the proportion of fallbacks in each IPv4 /8, /16, - # and /24 - def describe_fallback_ipv4_netblocks(self): - # this doesn't actually tell us anything useful - #self.describe_fallback_ipv4_netblock_mask(8) - self.describe_fallback_ipv4_netblock_mask(16) - self.describe_fallback_ipv4_netblock_mask(24) - - # log a message about the proportion of fallbacks in each IPv6 /12 (RIR), - # /23 (smaller RIR blocks), /32 (LIR), /48 (Customer), and /64 (Host) - # https://www.iana.org/assignments/ipv6-unicast-address-assignments/ - def describe_fallback_ipv6_netblocks(self): - # these don't actually tell us anything useful - #self.describe_fallback_ipv6_netblock_mask(12) - #self.describe_fallback_ipv6_netblock_mask(23) - self.describe_fallback_ipv6_netblock_mask(32) - self.describe_fallback_ipv6_netblock_mask(48) - self.describe_fallback_ipv6_netblock_mask(64) - - # log a message about the proportion of fallbacks in each IPv4 and IPv6 - # netblock - def describe_fallback_netblocks(self): - self.describe_fallback_ipv4_netblocks() - self.describe_fallback_ipv6_netblocks() - - # return a list of fallbacks which are on the IPv4 ORPort port - def fallbacks_on_ipv4_orport(self, port): - return filter(lambda x: x.orport == port, self.fallbacks) - - # return a list of fallbacks which are on the IPv6 ORPort port - def fallbacks_on_ipv6_orport(self, port): - return filter(lambda x: x.ipv6orport == port, self.fallbacks_with_ipv6()) - - # return a list of fallbacks which are on the DirPort port - def fallbacks_on_dirport(self, port): - return filter(lambda x: x.dirport == port, self.fallbacks) - - # log a message about the proportion of fallbacks on IPv4 ORPort port - # and return that count - def describe_fallback_ipv4_orport(self, port): - port_count = len(self.fallbacks_on_ipv4_orport(port)) - fallback_count = len(self.fallbacks) - logging.warning('%s of fallbacks are on IPv4 ORPort %d'%( - CandidateList.describe_percentage(port_count, - fallback_count), - port)) - return port_count - - # log a message about the proportion of IPv6 fallbacks on IPv6 ORPort port - # and return that count - def describe_fallback_ipv6_orport(self, port): - port_count = len(self.fallbacks_on_ipv6_orport(port)) - fallback_count = len(self.fallbacks_with_ipv6()) - logging.warning('%s of IPv6 fallbacks are on IPv6 ORPort %d'%( - CandidateList.describe_percentage(port_count, - fallback_count), - port)) - return port_count - - # log a message about the proportion of fallbacks on DirPort port - # and return that count - def describe_fallback_dirport(self, port): - port_count = len(self.fallbacks_on_dirport(port)) - fallback_count = len(self.fallbacks) - logging.warning('%s of fallbacks are on DirPort %d'%( - CandidateList.describe_percentage(port_count, - fallback_count), - port)) - return port_count - - # log a message about the proportion of fallbacks on each dirport, - # each IPv4 orport, and each IPv6 orport - def describe_fallback_ports(self): - fallback_count = len(self.fallbacks) - ipv4_or_count = fallback_count - ipv4_or_count -= self.describe_fallback_ipv4_orport(443) - ipv4_or_count -= self.describe_fallback_ipv4_orport(9001) - logging.warning('%s of fallbacks are on other IPv4 ORPorts'%( - CandidateList.describe_percentage(ipv4_or_count, - fallback_count))) - ipv6_fallback_count = len(self.fallbacks_with_ipv6()) - ipv6_or_count = ipv6_fallback_count - ipv6_or_count -= self.describe_fallback_ipv6_orport(443) - ipv6_or_count -= self.describe_fallback_ipv6_orport(9001) - logging.warning('%s of IPv6 fallbacks are on other IPv6 ORPorts'%( - CandidateList.describe_percentage(ipv6_or_count, - ipv6_fallback_count))) - dir_count = fallback_count - dir_count -= self.describe_fallback_dirport(80) - dir_count -= self.describe_fallback_dirport(9030) - logging.warning('%s of fallbacks are on other DirPorts'%( - CandidateList.describe_percentage(dir_count, - fallback_count))) - - # return a list of fallbacks which have the Exit flag - def fallbacks_with_exit(self): - return filter(lambda x: x.is_exit(), self.fallbacks) - - # log a message about the proportion of fallbacks with an Exit flag - def describe_fallback_exit_flag(self): - exit_falback_count = len(self.fallbacks_with_exit()) - fallback_count = len(self.fallbacks) - logging.warning('%s of fallbacks have the Exit flag'%( - CandidateList.describe_percentage(exit_falback_count, - fallback_count))) - - # return a list of fallbacks which have an IPv6 address - def fallbacks_with_ipv6(self): - return filter(lambda x: x.has_ipv6(), self.fallbacks) - - # log a message about the proportion of fallbacks on IPv6 - def describe_fallback_ip_family(self): - ipv6_falback_count = len(self.fallbacks_with_ipv6()) - fallback_count = len(self.fallbacks) - logging.warning('%s of fallbacks are on IPv6'%( - CandidateList.describe_percentage(ipv6_falback_count, - fallback_count))) - - def summarise_fallbacks(self, eligible_count, operator_count, failed_count, - guard_count, target_count): - s = '' - s += '/* To comment-out entries in this file, use C comments, and add *' - s += ' to the start of each line. (stem finds fallback entries using "' - s += ' at the start of a line.) */' - s += '\n' - # Report: - # whether we checked consensus download times - # the number of fallback directories (and limits/exclusions, if relevant) - # min & max fallback bandwidths - # #error if below minimum count - if PERFORM_IPV4_DIRPORT_CHECKS or PERFORM_IPV6_DIRPORT_CHECKS: - s += '/* Checked %s%s%s DirPorts served a consensus within %.1fs. */'%( - 'IPv4' if PERFORM_IPV4_DIRPORT_CHECKS else '', - ' and ' if (PERFORM_IPV4_DIRPORT_CHECKS - and PERFORM_IPV6_DIRPORT_CHECKS) else '', - 'IPv6' if PERFORM_IPV6_DIRPORT_CHECKS else '', - CONSENSUS_DOWNLOAD_SPEED_MAX) - else: - s += '/* Did not check IPv4 or IPv6 DirPort consensus downloads. */' - s += '\n' - # Multiline C comment with #error if things go bad - s += '/*' - s += '\n' - # Integers don't need escaping in C comments - fallback_count = len(self.fallbacks) - if FALLBACK_PROPORTION_OF_GUARDS is None: - fallback_proportion = '' - else: - fallback_proportion = ', Target %d (%d * %.2f)'%(target_count, - guard_count, - FALLBACK_PROPORTION_OF_GUARDS) - s += 'Final Count: %d (Eligible %d%s'%(fallback_count, eligible_count, - fallback_proportion) - if MAX_FALLBACK_COUNT is not None: - s += ', Max %d'%(MAX_FALLBACK_COUNT) - s += ')\n' - if eligible_count != fallback_count: - removed_count = eligible_count - fallback_count - excess_to_target_or_max = (eligible_count - operator_count - failed_count - - fallback_count) - # some 'Failed' failed the check, others 'Skipped' the check, - # if we already had enough successful downloads - s += ('Excluded: %d (Same Operator %d, Failed/Skipped Download %d, ' + - 'Excess %d)')%(removed_count, operator_count, failed_count, - excess_to_target_or_max) - s += '\n' - min_fb = self.fallback_min() - min_bw = min_fb._data['measured_bandwidth'] - max_fb = self.fallback_max() - max_bw = max_fb._data['measured_bandwidth'] - s += 'Bandwidth Range: %.1f - %.1f MB/s'%(min_bw/(1024.0*1024.0), - max_bw/(1024.0*1024.0)) - s += '\n' - s += '*/' - if fallback_count < MIN_FALLBACK_COUNT: - # We must have a minimum number of fallbacks so they are always - # reachable, and are in diverse locations - s += '\n' - s += '#error Fallback Count %d is too low. '%(fallback_count) - s += 'Must be at least %d for diversity. '%(MIN_FALLBACK_COUNT) - s += 'Try adding entries to the whitelist, ' - s += 'or setting INCLUDE_UNLISTED_ENTRIES = True.' - return s - -## Main Function - -def list_fallbacks(): - """ Fetches required onionoo documents and evaluates the - fallback directory criteria for each of the relays """ - - logging.warning('Downloading and parsing Onionoo data. ' + - 'This may take some time.') - # find relays that could be fallbacks - candidates = CandidateList() - candidates.add_relays() - - # work out how many fallbacks we want - guard_count = candidates.count_guards() - if FALLBACK_PROPORTION_OF_GUARDS is None: - target_count = guard_count - else: - target_count = int(guard_count * FALLBACK_PROPORTION_OF_GUARDS) - # the maximum number of fallbacks is the least of: - # - the target fallback count (FALLBACK_PROPORTION_OF_GUARDS * guard count) - # - the maximum fallback count (MAX_FALLBACK_COUNT) - if MAX_FALLBACK_COUNT is None: - max_count = target_count - else: - max_count = min(target_count, MAX_FALLBACK_COUNT) - - candidates.compute_fallbacks() - prefilter_fallbacks = copy.copy(candidates.fallbacks) - - # filter with the whitelist and blacklist - # if a relay has changed IPv4 address or ports recently, it will be excluded - # as ineligible before we call apply_filter_lists, and so there will be no - # warning that the details have changed from those in the whitelist. - # instead, there will be an info-level log during the eligibility check. - initial_count = len(candidates.fallbacks) - excluded_count = candidates.apply_filter_lists() - print candidates.summarise_filters(initial_count, excluded_count) - eligible_count = len(candidates.fallbacks) - - # calculate the measured bandwidth of each relay, - # then remove low-bandwidth relays - candidates.calculate_measured_bandwidth() - candidates.remove_low_bandwidth_relays() - - # print the raw fallback list - #for x in candidates.fallbacks: - # print x.fallbackdir_line(True) - # print json.dumps(candidates[x]._data, sort_keys=True, indent=4, - # separators=(',', ': '), default=json_util.default) - - # impose mandatory conditions here, like one per contact, family, IP - # in measured bandwidth order - candidates.sort_fallbacks_by_measured_bandwidth() - operator_count = 0 - # only impose these limits on the final list - operators can nominate - # multiple candidate fallbacks, and then we choose the best set - if not OUTPUT_CANDIDATES: - operator_count += candidates.limit_fallbacks_same_ip() - operator_count += candidates.limit_fallbacks_same_contact() - operator_count += candidates.limit_fallbacks_same_family() - - # check if each candidate can serve a consensus - # there's a small risk we've eliminated relays from the same operator that - # can serve a consensus, in favour of one that can't - # but given it takes up to 15 seconds to check each consensus download, - # the risk is worth it - if PERFORM_IPV4_DIRPORT_CHECKS or PERFORM_IPV6_DIRPORT_CHECKS: - logging.warning('Checking consensus download speeds. ' + - 'This may take some time.') - failed_count = candidates.perform_download_consensus_checks(max_count) - - # analyse and log interesting diversity metrics - # like netblock, ports, exit, IPv4-only - # (we can't easily analyse AS, and it's hard to accurately analyse country) - candidates.describe_fallback_ip_family() - # if we can't import the ipaddress module, we can't do netblock analysis - if HAVE_IPADDRESS: - candidates.describe_fallback_netblocks() - candidates.describe_fallback_ports() - candidates.describe_fallback_exit_flag() - - # output C comments summarising the fallback selection process - if len(candidates.fallbacks) > 0: - print candidates.summarise_fallbacks(eligible_count, operator_count, - failed_count, guard_count, - target_count) - else: - print '/* No Fallbacks met criteria */' - - # output C comments specifying the OnionOO data used to create the list - for s in fetch_source_list(): - print describe_fetch_source(s) - - # if we're outputting the final fallback list, sort by fingerprint - # this makes diffs much more stable - # otherwise, leave sorted by bandwidth, which allows operators to be - # contacted in priority order - if not OUTPUT_CANDIDATES: - candidates.sort_fallbacks_by_fingerprint() - - for x in candidates.fallbacks: - print x.fallbackdir_line(candidates.fallbacks, prefilter_fallbacks) - -if __name__ == "__main__": - list_fallbacks() diff --git a/scripts/maint/updateRustDependencies.sh b/scripts/maint/updateRustDependencies.sh new file mode 100755 index 0000000000..6d0587351f --- /dev/null +++ b/scripts/maint/updateRustDependencies.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2018 The Tor Project, Inc. +# Copyright (c) 2018 isis agora lovecruft +# See LICENSE for license information +# +# updateRustDependencies.sh +# ------------------------- +# Update our vendored Rust dependencies, either adding/removing +# dependencies and/or upgrading current dependencies to newer +# versions. +# +# To use this script, first add your dependencies, exactly specifying +# their versions, into the appropriate *crate-level* Cargo.toml in +# src/rust/ (i.e. *not* /src/rust/Cargo.toml, but instead the one for +# your crate). +# +# Next, run this script. Then, go into src/ext/rust and commit the +# changes to the tor-rust-dependencies repo. + +set -e + +HERE=$(dirname "$(realpath "$0")") +TOPLEVEL=$(dirname "$(dirname "$HERE")") +TOML="$TOPLEVEL/src/rust/Cargo.toml" +VENDORED="$TOPLEVEL/src/ext/rust/crates" +CARGO=$(command -v cargo) + +if ! test -f "$TOML" ; then + printf "Error: Couldn't find workspace Cargo.toml in expected location: %s\\n" "$TOML" +fi + +if ! test -d "$VENDORED" ; then + printf "Error: Couldn't find directory for Rust dependencies! Expected location: %s\\n" "$VENDORED" +fi + +if test -z "$CARGO" ; then + printf "Error: cargo must be installed and in your \$PATH\\n" +fi + +if test -z "$(cargo --list | grep vendor)" ; then + printf "Error: cargo-vendor not installed\\n" +fi + +$CARGO vendor -v --locked --explicit-version --no-delete --sync "$TOML" "$VENDORED" diff --git a/scripts/maint/updateVersions.pl.in b/scripts/maint/updateVersions.pl.in deleted file mode 100755 index 65c51a1f2d..0000000000 --- a/scripts/maint/updateVersions.pl.in +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/perl -w - -$CONFIGURE_IN = '@abs_top_srcdir@/configure.ac'; -$ORCONFIG_H = '@abs_top_srcdir@/src/win32/orconfig.h'; -$TOR_NSI = '@abs_top_srcdir@/contrib/win32build/tor-mingw.nsi.in'; - -$quiet = 1; - -sub demand { - my $fn = shift; - die "Missing file $fn" unless (-f $fn); -} - -demand($CONFIGURE_IN); -demand($ORCONFIG_H); -demand($TOR_NSI); - -# extract version from configure.ac - -open(F, $CONFIGURE_IN) or die "$!"; -$version = undef; -while (<F>) { - if (/AC_INIT\(\[tor\],\s*\[([^\]]*)\]\)/) { - $version = $1; - last; - } -} -die "No version found" unless $version; -print "Tor version is $version\n" unless $quiet; -close F; - -sub correctversion { - my ($fn, $defchar) = @_; - undef $/; - open(F, $fn) or die "$!"; - my $s = <F>; - close F; - if ($s =~ /^$defchar(?:)define\s+VERSION\s+\"([^\"]+)\"/m) { - $oldver = $1; - if ($oldver ne $version) { - print "Version mismatch in $fn: It thinks that the version is $oldver. I think it's $version. Fixing.\n"; - $line = $defchar . "define VERSION \"$version\""; - open(F, ">$fn.bak"); - print F $s; - close F; - $s =~ s/^$defchar(?:)define\s+VERSION.*?$/$line/m; - open(F, ">$fn"); - print F $s; - close F; - } else { - print "$fn has the correct version. Good.\n" unless $quiet; - } - } else { - print "Didn't find a version line in $fn -- uh oh.\n"; - } -} - -correctversion($TOR_NSI, "!"); -correctversion($ORCONFIG_H, "#"); diff --git a/scripts/maint/update_versions.py b/scripts/maint/update_versions.py new file mode 100755 index 0000000000..8067f2c6c8 --- /dev/null +++ b/scripts/maint/update_versions.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import io +import os +import re +import sys +import time + +def P(path): + """ + Give 'path' as a path relative to the abs_top_srcdir environment + variable. + """ + return os.path.join( + os.environ.get('abs_top_srcdir', "."), + path) + +def warn(msg): + """ + Print an warning message. + """ + print("WARNING: {}".format(msg), file=sys.stderr) + +def find_version(infile): + """ + Given an open file (or some other iterator of lines) holding a + configure.ac file, find the current version line. + """ + for line in infile: + m = re.search(r'AC_INIT\(\[tor\],\s*\[([^\]]*)\]\)', line) + if m: + return m.group(1) + + return None + +def update_version_in(infile, outfile, regex, versionline): + """ + Copy every line from infile to outfile. If any line matches 'regex', + replace it with 'versionline'. Return True if any line was changed; + false otherwise. + + 'versionline' is either a string -- in which case it is used literally, + or a function that receives the output of 'regex.match'. + """ + found = False + have_changed = False + for line in infile: + m = regex.match(line) + if m: + found = True + oldline = line + if type(versionline) == type(u""): + line = versionline + else: + line = versionline(m) + if not line.endswith("\n"): + line += "\n" + if oldline != line: + have_changed = True + outfile.write(line) + + if not found: + warn("didn't find any version line to replace in {}".format(infile.name)) + + return have_changed + +def replace_on_change(fname, change): + """ + If "change" is true, replace fname with fname.tmp. Otherwise, + delete fname.tmp. Log what we're doing to stderr. + """ + if not change: + print("No change in {}".format(fname)) + os.unlink(fname+".tmp") + else: + print("Updating {}".format(fname)) + os.rename(fname+".tmp", fname) + + +def update_file(fname, + regex, + versionline, + encoding="utf-8"): + """ + Replace any line matching 'regex' in 'fname' with 'versionline'. + Do not modify 'fname' if there are no changes made. Use the + provided encoding to read and write. + """ + with io.open(fname, "r", encoding=encoding) as f, \ + io.open(fname+".tmp", "w", encoding=encoding) as outf: + have_changed = update_version_in(f, outf, regex, versionline) + + replace_on_change(fname, have_changed) + +# Find out our version +with open("configure.ac") as f: + version = find_version(f) + +# If we have no version, we can't proceed. +if version == None: + print("No version found in configure.ac", file=sys.stderr()) + sys.exit(1) + +print("The version is {}".format(version)) + +today = time.strftime("%Y-%m-%d", time.gmtime()) + +# In configure.ac, we replace the definition of APPROX_RELEASE_DATE +# with "{today} for {version}", but only if the version does not match +# what is already there. +def replace_fn(m): + if m.group(1) != version: + # The version changed -- we change the date. + return u'AC_DEFINE(APPROX_RELEASE_DATE, ["{}"], # for {}'.format(today, version) + else: + # No changes. + return m.group(0) +update_file(P("configure.ac"), + re.compile(r'AC_DEFINE\(APPROX_RELEASE_DATE.* for (.*)'), + replace_fn) + +# In tor-mingw.nsi.in, we replace the definition of VERSION. +update_file(P("contrib/win32build/tor-mingw.nsi.in"), + re.compile(r'!define VERSION .*'), + u'!define VERSION "{}"'.format(version), + encoding="iso-8859-1") + +# In src/win32/orconfig.h, we replace the definition of VERSION. +update_file(P("src/win32/orconfig.h"), + re.compile(r'#define VERSION .*'), + u'#define VERSION "{}"'.format(version)) diff --git a/scripts/test/appveyor-irc-notify.py b/scripts/test/appveyor-irc-notify.py new file mode 100644 index 0000000000..cfe0afe7ae --- /dev/null +++ b/scripts/test/appveyor-irc-notify.py @@ -0,0 +1,219 @@ +# coding=utf8 +# Copyright (C) 2015-2016 Christopher R. Wood +# Copyright (c) 2018 The Tor Project +# Copyright (c) 2018 isis agora lovecruft +# +# From: https://raw.githubusercontent.com/gridsync/gridsync/def54f8166089b733d166665fdabcad4cdc526d8/misc/irc-notify.py +# and: https://github.com/gridsync/gridsync +# +# Modified by nexB on October 2016: +# - rework the handling of environment variables. +# - made the script use functions +# - support only Appveyor loading its environment variable to craft IRC notices. +# +# Modified by isis agora lovecruft <isis@torproject.org> in 2018: +# - Make IRC server configurable. +# - Make bot IRC nick deterministic. +# - Make bot join the channel rather than sending NOTICE messages externally. +# - Fix a bug which always caused sys.exit() to be logged as a traceback. +# - Actually reset the IRC colour codes after printing. +# +# Modified by Marcin Cieślak in 2018: +# - Accept UTF-8 +# - only guess github URLs +# - stop using ANSI colors +# +# Modified by teor in 2018: +# - fix github provider detection ('gitHub' or 'gitHubEnterprise', apparently) +# - make short commits 10 hexdigits long (that's what git does for tor) +# - generate correct branches and URLs for pull requests and tags +# - switch to one URL per line + +# This program is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software Foundation; +# either version 2 of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with this +# program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, +# Fifth Floor, Boston, MA 02110-1301 USA. + +"""Simple AppVeyor IRC notification script. + +The first argument is an IRC server and port; the second is the channel. Other +arguments passed to the script will be sent as notice messages content and any +{var}-formatted environment variables will be expanded automatically, replaced +with a corresponding Appveyor environment variable value. Use commas to +delineate multiple messages. + + +Example: +export APPVEYOR_ACCOUNT_NAME=isislovecruft +export APPVEYOR_BUILD_VERSION=1 +export APPVEYOR_PROJECT_NAME=tor +export APPVEYOR_PULL_REQUEST_NUMBER=pull_request_number +export APPVEYOR_PULL_REQUEST_TITLE=pull_request_title +export APPVEYOR_REPO_BRANCH=repo_branch +export APPVEYOR_REPO_COMMIT=22c95b72e29248dc4de9b85e590ee18f6f587de8 +export APPVEYOR_REPO_COMMIT_AUTHOR=isislovecruft +export APPVEYOR_REPO_COMMIT_MESSAGE="some IRC test" +export APPVEYOR_REPO_COMMIT_TIMESTAMP=2018-04-23 +export APPVEYOR_REPO_NAME=isislovecruft/tor +export APPVEYOR_REPO_PROVIDER=github +export APPVEYOR_URL=https://ci.appveyor.com +python ./appveyor-irc-notify.py irc.oftc.net:6697 tor-ci '{repo_name} {repo_branch} {short_commit} - {repo_commit_author}: {repo_commit_message}','Build #{build_version} passed. Details: {build_url} | Commit: {commit_url} + +See also https://github.com/gridsync/gridsync/blob/master/appveyor.yml for examples +in Appveyor's YAML: + + on_success: + - "python scripts/test/appveyor-irc-notify.py irc.oftc.net:6697 tor-ci success + on_failure: + - "python scripts/test/appveyor-irc-notify.py irc.oftc.net:6697 tor-ci failure +""" + +from __future__ import print_function +from __future__ import absolute_import + +import os +import random +import socket +import ssl +import sys +import time + + +def appveyor_vars(): + """ + Return a dict of key value crafted from appveyor environment variables. + """ + + vars = dict([ + ( + v.replace('APPVEYOR_', '').lower(), + os.getenv(v, '').decode('utf-8') + ) for v in [ + 'APPVEYOR_ACCOUNT_NAME', + 'APPVEYOR_BUILD_VERSION', + 'APPVEYOR_PROJECT_NAME', + 'APPVEYOR_PULL_REQUEST_HEAD_COMMIT', + 'APPVEYOR_PULL_REQUEST_HEAD_REPO_BRANCH', + 'APPVEYOR_PULL_REQUEST_HEAD_REPO_NAME', + 'APPVEYOR_PULL_REQUEST_NUMBER', + 'APPVEYOR_PULL_REQUEST_TITLE', + 'APPVEYOR_REPO_BRANCH', + 'APPVEYOR_REPO_COMMIT', + 'APPVEYOR_REPO_COMMIT_AUTHOR', + 'APPVEYOR_REPO_COMMIT_AUTHOR_EMAIL', + 'APPVEYOR_REPO_COMMIT_MESSAGE', + 'APPVEYOR_REPO_COMMIT_MESSAGE_EXTENDED', + 'APPVEYOR_REPO_COMMIT_TIMESTAMP', + 'APPVEYOR_REPO_NAME', + 'APPVEYOR_REPO_PROVIDER', + 'APPVEYOR_REPO_TAG_NAME', + 'APPVEYOR_URL', + ] + ]) + + BUILD_FMT = u'{url}/project/{account_name}/{project_name}/build/{build_version}' + + if vars["repo_tag_name"]: + BRANCH_FMT = u'{repo_name} {repo_tag_name} {short_commit}' + else: + BRANCH_FMT = u'{repo_name} {repo_branch} {short_commit}' + + vars.update(head_commit=vars["repo_commit"]) + + if vars["repo_provider"].lower().startswith('github'): + COMMIT_FMT = u'https://github.com/{repo_name}/commit/{repo_commit}' + if vars["pull_request_number"]: + vars.update(head_commit=vars["pull_request_head_commit"]) + BRANCH_FMT = u'{repo_name} {repo_branch} pull {pull_request_head_repo_name} {pull_request_head_repo_branch} {short_commit}' + COMMIT_FMT = u'https://github.com/{pull_request_head_repo_name}/commit/{pull_request_head_commit}' + PULL_FMT = u'https://github.com/{repo_name}/pull/{pull_request_number}' + vars.update(pull_url=PULL_FMT.format(**vars)) + vars.update(commit_url=COMMIT_FMT.format(**vars)) + + vars.update(short_commit=vars["head_commit"][:10]) + + vars.update( + build_url=BUILD_FMT.format(**vars), + branch_detail=BRANCH_FMT.format(**vars), + ) + return vars + + +def notify(): + """ + Send IRC notification + """ + apvy_vars = appveyor_vars() + + server, port = sys.argv[1].rsplit(":", 1) + channel = sys.argv[2] + success = sys.argv[3] == "success" + failure = sys.argv[3] == "failure" + + if success or failure: + messages = [] + messages.append(u"{branch_detail} - {repo_commit_author}: {repo_commit_message}") + + if success: + messages.append(u"Build #{build_version} passed. Details: {build_url}") + if failure: + messages.append(u"Build #{build_version} failed. Details: {build_url}") + + if "commit_url" in apvy_vars: + messages.append(u"Commit: {commit_url}") + + if "pull_url" in apvy_vars: + messages.append(u"Pull: {pull_url}") + + else: + messages = sys.argv[3:] + messages = ' '.join(messages) + messages = messages.decode("utf-8").split(',') + + print(repr(apvy_vars)) + messages = [msg.format(**apvy_vars).strip() for msg in messages] + + irc_username = 'appveyor-ci' + irc_nick = irc_username + + # establish connection + irc_sock = ssl.wrap_socket(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) + irc_sock.connect((socket.gethostbyname(server), int(port))) + irc_sock.send('NICK {0}\r\nUSER {0} * 0 :{0}\r\n'.format(irc_username).encode()) + irc_sock.send('JOIN #{0}\r\n'.format(channel).encode()) + irc_file = irc_sock.makefile() + + while irc_file: + line = irc_file.readline() + print(line.rstrip()) + response = line.split() + + if response[0] == 'PING': + irc_file.send('PONG {}\r\n'.format(response[1]).encode()) + + elif response[1] == '433': + irc_sock.send('NICK {}\r\n'.format(irc_nick).encode()) + + elif response[1] == '001': + time.sleep(5) + # send notification + for msg in messages: + print(u'PRIVMSG #{} :{}'.format(channel, msg).encode("utf-8")) + irc_sock.send(u'PRIVMSG #{} :{}\r\n'.format(channel, msg).encode("utf-8")) + time.sleep(5) + return + + +if __name__ == '__main__': + try: + notify() + except: + import traceback + print('ERROR: Failed to send notification: \n' + traceback.format_exc()) diff --git a/scripts/test/chutney-git-bisect.sh b/scripts/test/chutney-git-bisect.sh new file mode 100755 index 0000000000..dcf8ab1102 --- /dev/null +++ b/scripts/test/chutney-git-bisect.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +# Compile tor and run chutney to find out if the current commit works +# +# Usage: +# # Copy the script, so it doesn't change during bisection +# cp scripts/test/chutney-git-bisect.sh . +# git bisect run \ +# ./chutney-git-bisect.sh [tries [build-dir [flavour [skip-flavour]]]] +# +# Runs chutney up to <tries> times (default 3), because some bugs involve race +# conditions. +# Changes to <build-dir> (default no cd) before running tests. +# Runs chutney network <flavour> (default make test-network-all) as the test. +# Skips the test if <skip-flavour> fails (default no skip). + +CHUTNEY_TRIES=3 +if [ -n "$1" ]; then + CHUTNEY_TRIES="$1" +fi + +if [ -n "$2" ]; then + cd "$2" || exit +fi + +CHUTNEY_TEST_CMD="make test-network-all" +if [ -n "$3" ]; then + CHUTNEY_TEST_CMD="$CHUTNEY_PATH/tools/test-network.sh --flavour $3" +fi + +CHUTNEY_SKIP_ON_FAIL_CMD="true" +if [ -n "$4" ]; then + CHUTNEY_SKIP_ON_FAIL_CMD="$CHUTNEY_PATH/tools/test-network.sh --flavour $4" +fi + +CHUTNEY_BUILD_CMD_OR="make src/or/tor src/tools/tor-gencert" +CHUTNEY_BUILD_CMD_APP="make src/app/tor src/tools/tor-gencert" +if ! ( $CHUTNEY_BUILD_CMD_APP || $CHUTNEY_BUILD_CMD_OR ) ; then + echo "building '$CHUTNEY_BUILD_CMD_APP || $CHUTNEY_BUILD_CMD_OR' failed, skip" + exit 125 +fi + +if ! $CHUTNEY_SKIP_ON_FAIL_CMD ; then + echo "pre-condition '$CHUTNEY_SKIP_ON_FAIL_CMD' failed, skip" + exit 125 +fi + +i=1 +while [ "$i" -le "$CHUTNEY_TRIES" ]; do + echo + echo "Round $i/$CHUTNEY_TRIES:" + echo + if $CHUTNEY_TEST_CMD ; then + echo "test '$CHUTNEY_TEST_CMD' succeeded after $i/$CHUTNEY_TRIES attempts, good" + exit 0 + fi + i=$((i+1)) +done + +i=$((i-1)) +echo "test '$CHUTNEY_TEST_CMD' failed $i/$CHUTNEY_TRIES attempts, bad" +exit 1 diff --git a/scripts/test/cov-diff b/scripts/test/cov-diff index 7da7f0be9d..f3ca856888 100755 --- a/scripts/test/cov-diff +++ b/scripts/test/cov-diff @@ -7,11 +7,15 @@ DIRA="$1" DIRB="$2" -for A in $DIRA/*; do - B=$DIRB/`basename $A` - perl -pe 's/^\s*\!*\d+:/ 1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$A" > "$A.tmp" - perl -pe 's/^\s*\!*\d+:/ 1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$B" > "$B.tmp" - diff -u "$A.tmp" "$B.tmp" +for B in "$DIRB"/*; do + A=$DIRA/$(basename "$B") + if [ -f "$A" ]; then + perl -pe 's/^\s*\!*\d+(\*?):/ 1$1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$A" > "$A.tmp" + else + cat /dev/null > "$A.tmp" + fi + perl -pe 's/^\s*\!*\d+(\*?):/ 1$1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$B" > "$B.tmp" + diff -u "$A.tmp" "$B.tmp" |perl -pe 's/^((?:\+\+\+|---)(?:.*tmp))\s+.*/$1/;' rm "$A.tmp" "$B.tmp" done diff --git a/scripts/test/cov-exclude b/scripts/test/cov-exclude index 5117f11ec4..5cb9b1282d 100755 --- a/scripts/test/cov-exclude +++ b/scripts/test/cov-exclude @@ -26,3 +26,9 @@ if ($excluding or $exclude_this) { s{^\s*\#\#+:}{ x:}; s{^ (\s*)(\d+):}{$1!!!$2:}; } + +if (eof and $excluding) { + warn "Runaway LCOV_EXCL_START in $ARGV"; + $excluding = 0; +} + diff --git a/scripts/test/coverage b/scripts/test/coverage index f4ae475828..f61c83bc72 100755 --- a/scripts/test/coverage +++ b/scripts/test/coverage @@ -7,31 +7,31 @@ dst=$1 -for fn in src/or/*.c src/common/*.c; do - BN=`basename $fn` - DN=`dirname $fn` - F=`echo $BN | sed -e 's/\.c$//;'` +for fn in src/core/*/*.c src/feature/*/*.c src/app/*/*.c src/lib/*/*.c; do + BN=$(basename "$fn") + DN=$(dirname "$fn") + F=$(echo "$BN" | sed -e 's/\.c$//;') GC="${BN}.gcov" # Figure out the object file names - ONS=`echo ${DN}/src_*-${F}.o` - ONS_WILDCARD_LITERAL="${DN}/src_*-${F}.o" + ONS=$(echo "${DN}"/*testing_a-"${F}".o) + ONS_WILDCARD_LITERAL="${DN}/*testing_a-${F}.o" # If the wildcard didn't expand, no files if [ "$ONS" != "${ONS_WILDCARD_LITERAL}" ] then for on in $ONS; do # We should have a gcno file - GCNO=`echo $on | sed -e 's/\.o$/\.gcno/;'` - if [ -e $GCNO ] + GCNO=$(echo "$on" | sed -e 's/\.o$/\.gcno/;') + if [ -e "$GCNO" ] then # No need to test for gcda, since gcov assumes no execution # if it's absent - rm -f $GC - gcov -o $on $fn - if [ -e $GC ] + rm -f "$GC" + gcov -o "$on" "$fn" + if [ -e "$GC" ] then - if [ -n $dst ] + if [ -d "$dst" ] then - mv $GC $dst/$GC + mv "$GC" "$dst"/"$GC" fi else echo "gcov -o $on $fn didn't make a .gcov file" diff --git a/scripts/test/scan-build.sh b/scripts/test/scan-build.sh index 36e69e6d00..26e05ff101 100644..100755 --- a/scripts/test/scan-build.sh +++ b/scripts/test/scan-build.sh @@ -5,37 +5,81 @@ # This script is used for running a bunch of clang scan-build checkers # on Tor. +# These don't seem to cause false positives in our code, so let's turn +# them on. CHECKERS="\ - -disable-checker deadcode.DeadStores \ - -enable-checker alpha.core.CastSize \ + -enable-checker alpha.core.CallAndMessageUnInitRefArg \ -enable-checker alpha.core.CastToStruct \ + -enable-checker alpha.core.Conversion \ + -enable-checker alpha.core.FixedAddr \ -enable-checker alpha.core.IdenticalExpr \ + -enable-checker alpha.core.PointerArithm \ -enable-checker alpha.core.SizeofPtr \ - -enable-checker alpha.security.ArrayBoundV2 \ + -enable-checker alpha.core.TestAfterDivZero \ -enable-checker alpha.security.MallocOverflow \ -enable-checker alpha.security.ReturnPtrRange \ - -enable-checker alpha.unix.SimpleStream + -enable-checker alpha.unix.BlockInCriticalSection \ + -enable-checker alpha.unix.Chroot \ + -enable-checker alpha.unix.PthreadLock \ + -enable-checker alpha.unix.PthreadLock \ + -enable-checker alpha.unix.SimpleStream \ + -enable-checker alpha.unix.Stream \ -enable-checker alpha.unix.cstring.BufferOverlap \ -enable-checker alpha.unix.cstring.NotNullTerminated \ - -enable-checker alpha.unix.cstring.OutOfBounds \ - -enable-checker alpha.core.FixedAddr \ + -enable-checker valist.CopyToSelf \ + -enable-checker valist.Uninitialized \ + -enable-checker valist.Unterminated \ + -enable-checker security.FloatLoopCounter \ -enable-checker security.insecureAPI.strcpy \ - -enable-checker alpha.unix.PthreadLock \ - -enable-checker alpha.core.PointerArithm \ - -enable-checker alpha.core.TestAfterDivZero \ " +# shellcheck disable=SC2034 +# These have high false-positive rates. +EXTRA_CHECKERS="\ + -enable-checker alpha.security.ArrayBoundV2 \ + -enable-checker alpha.unix.cstring.OutOfBounds \ + -enable-checker alpha.core.CastSize \ +" + +# shellcheck disable=SC2034 +# These don't seem to generate anything useful +NOISY_CHECKERS="\ + -enable-checker alpha.clone.CloneChecker \ + -enable-checker alpha.deadcode.UnreachableCode \ +" + +if test "x$SCAN_BUILD_OUTPUT" != "x"; then + OUTPUTARG="-o $SCAN_BUILD_OUTPUT" +else + OUTPUTARG="" +fi + +# shellcheck disable=SC2086 scan-build \ $CHECKERS \ ./configure scan-build \ + make clean + +# Make this not get scanned for dead assignments, since it has lots of +# dead assignments we don't care about. +# shellcheck disable=SC2086 +scan-build \ $CHECKERS \ - make -j2 -k + -disable-checker deadcode.DeadStores \ + make -j5 -k ./src/ext/ed25519/ref10/libed25519_ref10.a +# shellcheck disable=SC2086 +scan-build \ + $CHECKERS $OUTPUTARG \ + make -j5 -k + +CHECKERS="\ +" # This one gives a false positive on every strcmp. # -enable-checker alpha.core.PointerSub # Needs work -# alpha.unix.MallocWithAnnotations ?? +# -enable-checker alpha.unix.MallocWithAnnotations |