diff options
39 files changed, 574 insertions, 129 deletions
diff --git a/changes/bug16389 b/changes/bug16389 new file mode 100644 index 0000000000..b7eb35034a --- /dev/null +++ b/changes/bug16389 @@ -0,0 +1,12 @@ + o Hidden Service Enhancement + Client now uses an introduction point failure cache to know when to + fetch or keep a descriptor in their cache. + + When fetching a descriptor, for every introduction points in it, we look + them up in the failure cache to know if we keep the descriptor or not. + For this to work, everytime an introduction points is discarded (ex: + receiving a NACK), we note it down in our introduction cache. If all + introduction points for an onion service are in our failure cache, we + discard the descriptor and fetch a new one. + + See rendcache.c for a detailed explanation of the cache's behavior. diff --git a/changes/bug16539 b/changes/bug16539 new file mode 100644 index 0000000000..8a0b6d251c --- /dev/null +++ b/changes/bug16539 @@ -0,0 +1,4 @@ + o Minor bugfixes (Ed25519): + - Fix a memory leak when reading router descriptors with + expired Ed25519 certificate. Fixes bug 16539; bugfix on 0.2.7.2-alpha. + diff --git a/changes/bug16741 b/changes/bug16741 new file mode 100644 index 0000000000..ce3dfdcfc7 --- /dev/null +++ b/changes/bug16741 @@ -0,0 +1,3 @@ + o Minor bugfixes: + - Check correctly for windows socket errors in the workqueue backend. + Fixes bug 16741; bugfix on 0.2.6.3-alpha. diff --git a/changes/bug16742 b/changes/bug16742 new file mode 100644 index 0000000000..2002cb7c72 --- /dev/null +++ b/changes/bug16742 @@ -0,0 +1,3 @@ + o Documentation: + - Recommend a 40 GB example AccountingMax in torrc.sample rather + than a 4 GB max. Closes ticket 16742. diff --git a/changes/decouple_init_keys b/changes/decouple_init_keys new file mode 100644 index 0000000000..7f48d2b9d3 --- /dev/null +++ b/changes/decouple_init_keys @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Move the client-only parts of init_keys() into a separate function. + Closes ticket 16763. diff --git a/changes/decouple_lost_owner b/changes/decouple_lost_owner new file mode 100644 index 0000000000..88adb18546 --- /dev/null +++ b/changes/decouple_lost_owner @@ -0,0 +1,4 @@ + o Code simplification and refactoring: + - Treat the loss of an owning controller as equivalent to a SIGTERM + signal. This removes a tiny amount of duplicated code, and simplifies + our callgraph. Closes ticekt 16788. diff --git a/changes/decouple_retry_directory b/changes/decouple_retry_directory new file mode 100644 index 0000000000..11f901b04f --- /dev/null +++ b/changes/decouple_retry_directory @@ -0,0 +1,6 @@ + o Code simplification and refactoring: + - Change the function that's called when we need to retry all downloads + so that it only reschedules the downloads to happen immediately, rather + than launching them all at once itself. This further simplifies + Tor's callgraph. + diff --git a/changes/feature16533 b/changes/feature16533 new file mode 100644 index 0000000000..e9fea94c7e --- /dev/null +++ b/changes/feature16533 @@ -0,0 +1,4 @@ + o Minor features (performance) + - Improve the runtime speed of Ed25519 signature verification by using + Ed25519-donna's batch verification support when there are a lot of + signatures to verify at once. Implements ticket 16533. diff --git a/changes/feature16535 b/changes/feature16535 new file mode 100644 index 0000000000..3df46b658a --- /dev/null +++ b/changes/feature16535 @@ -0,0 +1,4 @@ + o Minor features (performance) + - Improve the runtime speed of Ed25519 operations and Curve25519 keypair + generation when built targeting 32 bit x86 platforms with SSE2 + available. Implements ticket 16535. diff --git a/changes/move_formatting_functions b/changes/move_formatting_functions new file mode 100644 index 0000000000..4ad5806f23 --- /dev/null +++ b/changes/move_formatting_functions @@ -0,0 +1,3 @@ + o Code simplification and refactoring: + - Move some format-parsing functions out of crypto.c and + crypto_curve25519.c into crypto_format.c and/or util_format.c. diff --git a/scripts/maint/analyze_callgraph.py b/scripts/maint/analyze_callgraph.py index 69ae1287e4..b28460489a 100755 --- a/scripts/maint/analyze_callgraph.py +++ b/scripts/maint/analyze_callgraph.py @@ -13,7 +13,7 @@ class Parser: def enter_func(self, name): if self.infunc and not self.extern: self.calls.setdefault(self.infunc, set()).update( self.calledfns ) - + self.calledfns = set() self.infunc = name self.extern = False @@ -23,7 +23,7 @@ class Parser: self.extern = False self.calledfns = set() for line in inp: - m = re.match(r"Call graph node for function: '([^']+)'", line) + m = re.match(r"Call graph node for function: '([^']+)'", line) if m: self.enter_func(m.group(1)) continue @@ -42,18 +42,28 @@ class Parser: def transitive_closure(g): + passno = 0 changed = True g = copy.deepcopy(g) + import random while changed: + passno += 1 changed = False - print "X" - for k in g.keys(): + keys = g.keys() + idx = 0 + for k in keys: + idx += 1 + print "Pass %d/?: %d/%d\r" %(passno, idx, len(keys)), + sys.stdout.flush() newset = g[k].copy() for fn in g[k]: newset.update(g.get(fn, set())) if len(newset) != len(g[k]): g[k].update( newset ) changed = True + + print + return g def strongly_connected_components(g): @@ -96,22 +106,114 @@ def strongly_connected_components(g): return all_sccs +def biggest_component(sccs): + return max(len(c) for c in sccs) + +def connection_bottlenecks(callgraph): + + callers = {} + for fn in callgraph: + for fn2 in callgraph[fn]: + callers.setdefault(fn2, set()).add(fn) + + components = strongly_connected_components(callgraph) + components.sort(key=len) + big_component_fns = components[-1] + size = len(big_component_fns) + + function_bottlenecks = fn_results = [] + + total = len(big_component_fns) + idx = 0 + for fn in big_component_fns: + idx += 1 + print "Pass 1/3: %d/%d\r"%(idx, total), + sys.stdout.flush() + cg2 = copy.deepcopy(callgraph) + del cg2[fn] + + fn_results.append( (size - biggest_component(strongly_connected_components(cg2)), fn) ) + + print + bcf_set = set(big_component_fns) + + call_bottlenecks = fn_results = [] + result_set = set() + total = len(big_component_fns) + idx = 0 + for fn in big_component_fns: + fn_callers = callers[fn].intersection(bcf_set) + idx += 1 + if len(fn_callers) != 1: + continue + + print "Pass 2/3: %d/%d\r"%(idx, total), + sys.stdout.flush() + + caller = fn_callers.pop() + assert len(fn_callers) == 0 + cg2 = copy.deepcopy(callgraph) + cg2[caller].remove(fn) + + fn_results.append( (size - biggest_component(strongly_connected_components(cg2)), fn, "called by", caller) ) + result_set.add( (caller, fn) ) + + print + + total = len(big_component_fns) + idx = 0 + for fn in big_component_fns: + fn_calls = callgraph[fn].intersection(bcf_set) + idx += 1 + if len(fn_calls) != 1: + continue + + print "Pass 3/3: %d/%d\r"%(idx, total), + sys.stdout.flush() + + callee = fn_calls.pop() + if (fn, callee) in result_set: + continue + + assert len(fn_calls) == 0 + cg2 = copy.deepcopy(callgraph) + cg2[fn].remove(callee) + + fn_results.append( (size - biggest_component(strongly_connected_components(cg2)), callee, "called by", fn) ) + + print + + + return (function_bottlenecks, call_bottlenecks) + if __name__ == '__main__': p = Parser() for fname in sys.argv[1:]: with open(fname, 'r') as f: p.parse_callgraph_file(f) + + sys.stdout.flush + + print "Building callgraph" callgraph = p.extract_callgraph() + print "Finding strongly connected components" sccs = strongly_connected_components(callgraph) + print "Finding the transitive closure of the callgraph.." closure = transitive_closure(callgraph) + print "Finding bottlenecks..." + bottlenecks = connection_bottlenecks(callgraph) + + data = { + 'callgraph' : callgraph, + 'sccs' : sccs, + 'closure' : closure, + 'bottlenecks' : bottlenecks } + with open('callgraph.pkl', 'w') as f: - cPickle.dump(callgraph, f) + cPickle.dump(data, f) + - with open('callgraph_closure.pkl', 'w') as f: - cPickle.dump(closure, f) - with open('callgraph_sccs.pkl', 'w') as f: - cPickle.dump(sccs, f) diff --git a/scripts/maint/display_callgraph.py b/scripts/maint/display_callgraph.py index 9ead56c8cc..211bfda28d 100755 --- a/scripts/maint/display_callgraph.py +++ b/scripts/maint/display_callgraph.py @@ -2,9 +2,12 @@ import cPickle -callgraph = cPickle.load(open("callgraph.pkl")) -closure = cPickle.load(open("callgraph_closure.pkl")) -sccs = cPickle.load(open("callgraph_sccs.pkl")) +data = cPickle.load(open("callgraph.pkl")) + +callgraph = data['callgraph'] +closure = data['closure'] +sccs = data['sccs'] +fn_bottle, call_bottle = data['bottlenecks'] for n_reachable, fn in sorted(list((len(r), fn) for fn, r in closure.iteritems())): print "%s can reach %s other functions." %(fn, n_reachable) @@ -13,10 +16,24 @@ for n_reachable, fn in sorted(list((len(r), fn) for fn, r in closure.iteritems() c = [ (len(component), component) for component in sccs ] c.sort() +print "\n================================" + for n, component in c: if n < 2: continue - print n, component + print "Strongly connected component of size %d:"%n + print component + + +print "\n================================" +print "====== Number of functions pulled into blob, by function in blob." +fn_bottle.sort() +for n, fn in fn_bottle[-30:]: + print "%3d: %s"%(n, fn) +print "====== Number of functions pulled into blob, by call in blob." +call_bottle.sort() +for n, fn1, _, fn2 in call_bottle[-30:]: + print "%3d: %s -> %s "%(n, fn2, fn1) diff --git a/src/common/compat.h b/src/common/compat.h index d3b18eba92..c7c468c754 100644 --- a/src/common/compat.h +++ b/src/common/compat.h @@ -563,6 +563,8 @@ int network_init(void); ((e) == WSAEMFILE || (e) == WSAENOBUFS) /** Return true if e is EADDRINUSE or the local equivalent. */ #define ERRNO_IS_EADDRINUSE(e) ((e) == WSAEADDRINUSE) +/** Return true if e is EINTR or the local equivalent */ +#define ERRNO_IS_EINTR(e) ((e) == WSAEINTR || 0) int tor_socket_errno(tor_socket_t sock); const char *tor_socket_strerror(int e); #else @@ -573,6 +575,7 @@ const char *tor_socket_strerror(int e); #else #define ERRNO_IS_EAGAIN(e) ((e) == EAGAIN || (e) == EWOULDBLOCK) #endif +#define ERRNO_IS_EINTR(e) ((e) == EINTR || 0) #define ERRNO_IS_EINPROGRESS(e) ((e) == EINPROGRESS || 0) #define ERRNO_IS_CONN_EINPROGRESS(e) ((e) == EINPROGRESS || 0) #define ERRNO_IS_ACCEPT_EAGAIN(e) \ diff --git a/src/common/compat_threads.c b/src/common/compat_threads.c index 15648d2851..85ad737574 100644 --- a/src/common/compat_threads.c +++ b/src/common/compat_threads.c @@ -88,7 +88,7 @@ in_main_thread(void) } #if defined(HAVE_EVENTFD) || defined(HAVE_PIPE) -/* non-interruptable versions */ +/* As write(), but retry on EINTR */ static int write_ni(int fd, const void *buf, size_t n) { @@ -99,6 +99,7 @@ write_ni(int fd, const void *buf, size_t n) goto again; return r; } +/* As read(), but retry on EINTR */ static int read_ni(int fd, void *buf, size_t n) { @@ -111,30 +112,32 @@ read_ni(int fd, void *buf, size_t n) } #endif -/* non-interruptable versions */ +/** As send(), but retry on EINTR. */ static int send_ni(int fd, const void *buf, size_t n, int flags) { int r; again: r = (int) send(fd, buf, n, flags); - if (r < 0 && errno == EINTR) + if (r < 0 && ERRNO_IS_EINTR(tor_socket_errno(fd))) goto again; return r; } +/** As recv(), but retry on EINTR. */ static int recv_ni(int fd, void *buf, size_t n, int flags) { int r; again: - r = (int) recv(fd, buf, n, flags); - if (r < 0 && errno == EINTR) + r = (int) recv(fd, buf, n, flags); + if (r < 0 && ERRNO_IS_EINTR(tor_socket_errno(fd))) goto again; return r; } #ifdef HAVE_EVENTFD +/* Increment the event count on an eventfd <b>fd</b> */ static int eventfd_alert(int fd) { @@ -145,6 +148,7 @@ eventfd_alert(int fd) return 0; } +/* Drain all events from an eventfd <b>fd</b>. */ static int eventfd_drain(int fd) { @@ -157,6 +161,7 @@ eventfd_drain(int fd) #endif #ifdef HAVE_PIPE +/** Send a byte over a pipe. Return 0 on success or EAGAIN; -1 on error */ static int pipe_alert(int fd) { @@ -166,6 +171,8 @@ pipe_alert(int fd) return 0; } +/** Drain all input from a pipe <b>fd</b> and ignore it. Return 0 on + * success, -1 on error. */ static int pipe_drain(int fd) { @@ -181,6 +188,8 @@ pipe_drain(int fd) } #endif +/** Send a byte on socket <b>fd</b>t. Return 0 on success or EAGAIN, + * -1 on error. */ static int sock_alert(tor_socket_t fd) { @@ -190,6 +199,8 @@ sock_alert(tor_socket_t fd) return 0; } +/** Drain all the input from a socket <b>fd</b>, and ignore it. Return 0 on + * success, -1 on error. */ static int sock_drain(tor_socket_t fd) { diff --git a/src/common/container.c b/src/common/container.c index 082afb51ee..636dfb6c57 100644 --- a/src/common/container.c +++ b/src/common/container.c @@ -742,7 +742,7 @@ smartlist_sort_strings(smartlist_t *sl) } /** Return the most frequent string in the sorted list <b>sl</b> */ -char * +const char * smartlist_get_most_frequent_string(smartlist_t *sl) { return smartlist_get_most_frequent(sl, compare_string_ptrs_); @@ -752,7 +752,7 @@ smartlist_get_most_frequent_string(smartlist_t *sl) * If <b>count_out</b> is provided, set <b>count_out</b> to the * number of times that string appears. */ -char * +const char * smartlist_get_most_frequent_string_(smartlist_t *sl, int *count_out) { return smartlist_get_most_frequent_(sl, compare_string_ptrs_, count_out); @@ -1020,7 +1020,7 @@ smartlist_sort_digests256(smartlist_t *sl) /** Return the most frequent member of the sorted list of DIGEST256_LEN * digests in <b>sl</b> */ -char * +const uint8_t * smartlist_get_most_frequent_digest256(smartlist_t *sl) { return smartlist_get_most_frequent(sl, compare_digests256_); diff --git a/src/common/container.h b/src/common/container.h index 2a6ba01e62..5abd8b48d9 100644 --- a/src/common/container.h +++ b/src/common/container.h @@ -109,9 +109,9 @@ void smartlist_sort_digests(smartlist_t *sl); void smartlist_sort_digests256(smartlist_t *sl); void smartlist_sort_pointers(smartlist_t *sl); -char *smartlist_get_most_frequent_string(smartlist_t *sl); -char *smartlist_get_most_frequent_string_(smartlist_t *sl, int *count_out); -char *smartlist_get_most_frequent_digest256(smartlist_t *sl); +const char *smartlist_get_most_frequent_string(smartlist_t *sl); +const char *smartlist_get_most_frequent_string_(smartlist_t *sl, int *count_out); +const uint8_t *smartlist_get_most_frequent_digest256(smartlist_t *sl); void smartlist_uniq_strings(smartlist_t *sl); void smartlist_uniq_digests(smartlist_t *sl); diff --git a/src/common/crypto.c b/src/common/crypto.c index 2121383c75..6d4b0d7e16 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -2365,11 +2365,20 @@ crypto_seed_rng(void) } /** Write <b>n</b> bytes of strong random data to <b>to</b>. Return 0 on - * success, -1 on failure. + * success, -1 on failure, with support for mocking for unit tests. */ MOCK_IMPL(int, crypto_rand, (char *to, size_t n)) { + return crypto_rand_unmocked(to, n); +} + +/** Write <b>n</b> bytes of strong random data to <b>to</b>. Return 0 on + * success, -1 on failure. Most callers will want crypto_rand instead. + */ +int +crypto_rand_unmocked(char *to, size_t n) +{ int r; tor_assert(n < INT_MAX); tor_assert(to); diff --git a/src/common/crypto.h b/src/common/crypto.h index 368e9d8e32..6256f7346b 100644 --- a/src/common/crypto.h +++ b/src/common/crypto.h @@ -260,6 +260,7 @@ int crypto_expand_key_material_rfc5869_sha256( /* random numbers */ int crypto_seed_rng(void); MOCK_DECL(int,crypto_rand,(char *to, size_t n)); +int crypto_rand_unmocked(char *to, size_t n); int crypto_strongest_rand(uint8_t *out, size_t out_len); int crypto_rand_int(unsigned int max); int crypto_rand_int_range(unsigned int min, unsigned int max); diff --git a/src/common/crypto_ed25519.c b/src/common/crypto_ed25519.c index 3a629322cb..7e995f4616 100644 --- a/src/common/crypto_ed25519.c +++ b/src/common/crypto_ed25519.c @@ -37,6 +37,8 @@ typedef struct { unsigned char *); int (*sign)(unsigned char *, const unsigned char *, size_t, const unsigned char *, const unsigned char *); + int (*open_batch)(const unsigned char **, size_t *, const unsigned char **, + const unsigned char **, size_t, int *); int (*blind_secret_key)(unsigned char *, const unsigned char *, const unsigned char *); @@ -57,6 +59,7 @@ static const ed25519_impl_t impl_ref10 = { ed25519_ref10_open, ed25519_ref10_sign, + NULL, ed25519_ref10_blind_secret_key, ed25519_ref10_blind_public_key, @@ -74,6 +77,7 @@ static const ed25519_impl_t impl_donna = { ed25519_donna_open, ed25519_donna_sign, + ed25519_sign_open_batch_donna, ed25519_donna_blind_secret_key, ed25519_donna_blind_public_key, @@ -197,57 +201,69 @@ ed25519_checksig_batch(int *okay_out, const ed25519_checkable_t *checkable, int n_checkable) { - int res, i; - - res = 0; - for (i = 0; i < n_checkable; ++i) { - const ed25519_checkable_t *ch = &checkable[i]; - int r = ed25519_checksig(&ch->signature, ch->msg, ch->len, ch->pubkey); - if (r < 0) - --res; - if (okay_out) - okay_out[i] = (r == 0); - } - -#if 0 - /* This is how we'd do it if we were using ed25519_donna. I'll keep this - * code around here in case we ever do that. */ - const uint8_t **ms; - size_t *lens; - const uint8_t **pks; - const uint8_t **sigs; - int *oks; - - ms = tor_malloc(sizeof(uint8_t*)*n_checkable); - lens = tor_malloc(sizeof(size_t)*n_checkable); - pks = tor_malloc(sizeof(uint8_t*)*n_checkable); - sigs = tor_malloc(sizeof(uint8_t*)*n_checkable); - oks = okay_out ? okay_out : tor_malloc(sizeof(int)*n_checkable); - - for (i = 0; i < n_checkable; ++i) { - ms[i] = checkable[i].msg; - lens[i] = checkable[i].len; - pks[i] = checkable[i].pubkey->pubkey; - sigs[i] = checkable[i].signature.sig; - oks[i] = 0; - } - - ed25519_sign_open_batch_donna_fb(ms, lens, pks, sigs, n_checkable, oks); + int i, res; + const ed25519_impl_t *impl = get_ed_impl(); - res = 0; - for (i = 0; i < n_checkable; ++i) { - /* XXX/yawning: Propagate to okay_out? */ - if (!oks[i]) - --res; + if (impl->open_batch == NULL) { + /* No batch verification implementation available, fake it by checking the + * each signature individually. + */ + res = 0; + for (i = 0; i < n_checkable; ++i) { + const ed25519_checkable_t *ch = &checkable[i]; + int r = ed25519_checksig(&ch->signature, ch->msg, ch->len, ch->pubkey); + if (r < 0) + --res; + if (okay_out) + okay_out[i] = (r == 0); + } + } else { + /* ed25519-donna style batch verification available. + * + * Theoretically, this should only be called if n_checkable >= 3, since + * that's the threshold where the batch verification actually kicks in, + * but the only difference is a few mallocs/frees. + */ + const uint8_t **ms; + size_t *lens; + const uint8_t **pks; + const uint8_t **sigs; + int *oks; + int all_ok; + + ms = tor_malloc(sizeof(uint8_t*)*n_checkable); + lens = tor_malloc(sizeof(size_t)*n_checkable); + pks = tor_malloc(sizeof(uint8_t*)*n_checkable); + sigs = tor_malloc(sizeof(uint8_t*)*n_checkable); + oks = okay_out ? okay_out : tor_malloc(sizeof(int)*n_checkable); + + for (i = 0; i < n_checkable; ++i) { + ms[i] = checkable[i].msg; + lens[i] = checkable[i].len; + pks[i] = checkable[i].pubkey->pubkey; + sigs[i] = checkable[i].signature.sig; + oks[i] = 0; + } + + res = 0; + all_ok = impl->open_batch(ms, lens, pks, sigs, n_checkable, oks); + for (i = 0; i < n_checkable; ++i) { + if (!oks[i]) + --res; + } + /* XXX: For now sanity check oks with the return value. Once we have + * more confidence in the code, if `all_ok == 0` we can skip iterating + * over oks since all the signatures were found to be valid. + */ + tor_assert(((res == 0) && !all_ok) || ((res < 0) && all_ok)); + + tor_free(ms); + tor_free(lens); + tor_free(pks); + if (! okay_out) + tor_free(oks); } - tor_free(ms); - tor_free(lens); - tor_free(pks); - if (! okay_out) - tor_free(oks); -#endif - return res; } diff --git a/src/common/workqueue.c b/src/common/workqueue.c index cf63634076..b0b004dc25 100644 --- a/src/common/workqueue.c +++ b/src/common/workqueue.c @@ -480,7 +480,8 @@ replyqueue_process(replyqueue_t *queue) if (queue->alert.drain_fn(queue->alert.read_fd) < 0) { static ratelim_t warn_limit = RATELIM_INIT(7200); log_fn_ratelim(&warn_limit, LOG_WARN, LD_GENERAL, - "Failure from drain_fd"); + "Failure from drain_fd: %s", + tor_socket_strerror(tor_socket_errno(queue->alert.read_fd))); } tor_mutex_acquire(&queue->lock); diff --git a/src/config/torrc.minimal.in-staging b/src/config/torrc.minimal.in-staging index bde800fd23..d54a5599cd 100644 --- a/src/config/torrc.minimal.in-staging +++ b/src/config/torrc.minimal.in-staging @@ -110,11 +110,11 @@ ## Use these to restrict the maximum traffic per day, week, or month. ## Note that this threshold applies separately to sent and received bytes, -## not to their sum: setting "4 GB" may allow up to 8 GB total before +## not to their sum: setting "40 GB" may allow up to 80 GB total before ## hibernating. ## -## Set a maximum of 4 gigabytes each way per period. -#AccountingMax 4 GBytes +## Set a maximum of 40 gigabytes each way per period. +#AccountingMax 40 GBytes ## Each period starts daily at midnight (AccountingMax is per day) #AccountingStart day 00:00 ## Each period starts on the 3rd of the month at 15:00 (AccountingMax diff --git a/src/config/torrc.sample.in b/src/config/torrc.sample.in index bde800fd23..d54a5599cd 100644 --- a/src/config/torrc.sample.in +++ b/src/config/torrc.sample.in @@ -110,11 +110,11 @@ ## Use these to restrict the maximum traffic per day, week, or month. ## Note that this threshold applies separately to sent and received bytes, -## not to their sum: setting "4 GB" may allow up to 8 GB total before +## not to their sum: setting "40 GB" may allow up to 80 GB total before ## hibernating. ## -## Set a maximum of 4 gigabytes each way per period. -#AccountingMax 4 GBytes +## Set a maximum of 40 gigabytes each way per period. +#AccountingMax 40 GBytes ## Each period starts daily at midnight (AccountingMax is per day) #AccountingStart day 00:00 ## Each period starts on the 3rd of the month at 15:00 (AccountingMax diff --git a/src/ext/ed25519/donna/README.tor b/src/ext/ed25519/donna/README.tor index 2bb0efc012..026d180c24 100644 --- a/src/ext/ed25519/donna/README.tor +++ b/src/ext/ed25519/donna/README.tor @@ -40,3 +40,7 @@ as of 8757bd4cd209cb032853ece0ce413f122eef212c. * On non-x86 targets, GCC's Stack Protector dislikes variables that have alignment constraints greater than that of other primitive types. The `ALIGN` macro is thus no-oped for all non-SSE2 builds. + + * On 32 bit x86 targets that the compiler thinks supports SSE2, always + enable SSE2 support by force defining ED25519_SSE2 (x86_64 would also + always support this, but that code path is slower). diff --git a/src/ext/ed25519/donna/ed25519-donna-portable.h b/src/ext/ed25519/donna/ed25519-donna-portable.h index 9ec83b87e3..9c9c55577c 100644 --- a/src/ext/ed25519/donna/ed25519-donna-portable.h +++ b/src/ext/ed25519/donna/ed25519-donna-portable.h @@ -144,6 +144,16 @@ static inline void U64TO8_LE(unsigned char *p, const uint64_t v) { #endif #endif +/* Tor: Force enable SSE2 on 32 bit x86 systems if the compile target + * architecture supports it. This is not done on x86-64 as the non-SSE2 + * code benchmarks better, at least on Haswell. + */ +#if defined(__SSE2__) /* && !defined(CPU_X86_64) */ + /* undef in case it's manually specified... */ + #undef ED25519_SSE2 + #define ED25519_SSE2 +#endif + /* Tor: GCC's Stack Protector freaks out and produces variable length * buffer warnings when alignment is requested that is greater than * STACK_BOUNDARY (x86 has special code to deal with this for SSE2). diff --git a/src/ext/ed25519/donna/ed25519-randombytes-custom.h b/src/ext/ed25519/donna/ed25519-randombytes-custom.h index e49368bbaf..3fb0959fc4 100644 --- a/src/ext/ed25519/donna/ed25519-randombytes-custom.h +++ b/src/ext/ed25519/donna/ed25519-randombytes-custom.h @@ -13,5 +13,5 @@ static void ED25519_FN(ed25519_randombytes_unsafe) (void *p, size_t len) { - crypto_rand(p, len); + crypto_rand_unmocked(p, len); } diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index bf7f8daca7..716024df6a 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -716,8 +716,8 @@ origin_circuit_new(void) return circ; } -/** Allocate a new or_circuit_t, connected to <b>p_conn</b> as - * <b>p_circ_id</b>. If <b>p_conn</b> is NULL, the circuit is unattached. */ +/** Allocate a new or_circuit_t, connected to <b>p_chan</b> as + * <b>p_circ_id</b>. If <b>p_chan</b> is NULL, the circuit is unattached. */ or_circuit_t * or_circuit_new(circid_t p_circ_id, channel_t *p_chan) { diff --git a/src/or/command.c b/src/or/command.c index 719b10736b..af6e0533d8 100644 --- a/src/or/command.c +++ b/src/or/command.c @@ -340,7 +340,6 @@ command_process_create_cell(cell_t *cell, channel_t *chan) if (len < 0) { log_warn(LD_OR,"Failed to generate key material. Closing."); circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL); - tor_free(create_cell); return; } created_cell.cell_type = CELL_CREATED_FAST; diff --git a/src/or/control.c b/src/or/control.c index a0004c3dbe..a5073037ab 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -3932,10 +3932,6 @@ connection_control_reached_eof(control_connection_t *conn) return 0; } -static void lost_owning_controller(const char *owner_type, - const char *loss_manner) - ATTR_NORETURN; - /** Shut down this Tor instance in the same way that SIGINT would, but * with a log message appropriate for the loss of an owning controller. */ static void @@ -3944,10 +3940,7 @@ lost_owning_controller(const char *owner_type, const char *loss_manner) log_notice(LD_CONTROL, "Owning controller %s has %s -- exiting now.", owner_type, loss_manner); - /* XXXX Perhaps this chunk of code should be a separate function, - * called here and by process_signal(SIGINT). */ - tor_cleanup(); - exit(0); + activate_signal(SIGTERM); } /** Called when <b>conn</b> is being freed. */ @@ -5552,8 +5545,6 @@ static char *owning_controller_process_spec = NULL; * if this Tor instance is not currently owned by a process. */ static tor_process_monitor_t *owning_controller_process_monitor = NULL; -static void owning_controller_procmon_cb(void *unused) ATTR_NORETURN; - /** Process-termination monitor callback for Tor's owning controller * process. */ static void diff --git a/src/or/dirserv.c b/src/or/dirserv.c index e70b1b422d..8eeba54374 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -62,7 +62,7 @@ static uint32_t dirserv_get_status_impl(const char *fp, const char *nickname, uint32_t addr, uint16_t or_port, const char *platform, const char **msg, - int should_log); + int severity); static void clear_cached_dir(cached_dir_t *d); static const signed_descriptor_t *get_signed_descriptor_by_fp( const char *fp, @@ -290,7 +290,7 @@ dirserv_router_get_status(const routerinfo_t *router, const char **msg, return dirserv_get_status_impl(d, router->nickname, router->addr, router->or_port, - router->platform, msg, 1); + router->platform, msg, severity); } /** Return true if there is no point in downloading the router described by diff --git a/src/or/dirvote.c b/src/or/dirvote.c index 0f3b77fe28..d8e6ee2229 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -478,7 +478,7 @@ compute_routerstatus_consensus(smartlist_t *votes, int consensus_method, if (microdesc_digest256_out) { smartlist_t *digests = smartlist_new(); - const char *best_microdesc_digest; + const uint8_t *best_microdesc_digest; SMARTLIST_FOREACH_BEGIN(votes, vote_routerstatus_t *, rs) { char d[DIGEST256_LEN]; if (compare_vote_rs(rs, most)) diff --git a/src/or/main.c b/src/or/main.c index 1c1c4774b4..06fc541567 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -1281,6 +1281,17 @@ reschedule_descriptor_update_check(void) time_to.check_descriptor = 0; } +/** + * Update our schedule so that we'll check whether we need to fetch directory + * info immediately. + */ +void +reschedule_directory_downloads(void) +{ + time_to.download_networkstatus = 0; + time_to.try_getting_descriptors = 0; +} + /** Perform regular maintenance tasks. This function gets run once per * second by second_elapsed_callback(). */ @@ -1913,7 +1924,7 @@ ip_address_changed(int at_interface) if (at_interface) { if (! server) { /* Okay, change our keys. */ - if (init_keys()<0) + if (init_keys_client() < 0) log_warn(LD_GENERAL, "Unable to rotate keys after IP change!"); } } else { diff --git a/src/or/main.h b/src/or/main.h index f31849f73e..447d3f4eca 100644 --- a/src/or/main.h +++ b/src/or/main.h @@ -54,6 +54,7 @@ void ip_address_changed(int at_interface); void dns_servers_relaunch_checks(void); void reset_all_main_loop_timers(void); void reschedule_descriptor_update_check(void); +void reschedule_directory_downloads(void); MOCK_DECL(long,get_uptime,(void)); diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 051603d327..a9bab3ddc6 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -132,7 +132,6 @@ get_microdesc_cache_noload(void) { if (PREDICT_UNLIKELY(the_microdesc_cache==NULL)) { microdesc_cache_t *cache = tor_malloc_zero(sizeof(*cache)); - tor_malloc_zero(sizeof(microdesc_cache_t)); HT_INIT(microdesc_map, &cache->map); cache->cache_fname = get_datadir_fname("cached-microdescs"); cache->journal_fname = get_datadir_fname("cached-microdescs.new"); diff --git a/src/or/router.c b/src/or/router.c index 47825e2d1c..03973ae90a 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -767,6 +767,46 @@ router_write_fingerprint(int hashed) return result; } +static int +init_keys_common(void) +{ + if (!key_lock) + key_lock = tor_mutex_new(); + + /* There are a couple of paths that put us here before we've asked + * openssl to initialize itself. */ + if (crypto_global_init(get_options()->HardwareAccel, + get_options()->AccelName, + get_options()->AccelDir)) { + log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting."); + return -1; + } + + return 0; +} + +int +init_keys_client(void) +{ + crypto_pk_t *prkey; + if (init_keys_common() < 0) + return -1; + + if (!(prkey = crypto_pk_new())) + return -1; + if (crypto_pk_generate_key(prkey)) { + crypto_pk_free(prkey); + return -1; + } + set_client_identity_key(prkey); + /* Create a TLS context. */ + if (router_initialize_tls_context() < 0) { + log_err(LD_GENERAL,"Error creating TLS context for Tor client."); + return -1; + } + return 0; +} + /** Initialize all OR private keys, and the TLS context, as necessary. * On OPs, this only initializes the tls context. Return 0 on success, * or -1 if Tor should die. @@ -786,35 +826,13 @@ init_keys(void) int v3_digest_set = 0; authority_cert_t *cert = NULL; - if (!key_lock) - key_lock = tor_mutex_new(); - - /* There are a couple of paths that put us here before we've asked - * openssl to initialize itself. */ - if (crypto_global_init(get_options()->HardwareAccel, - get_options()->AccelName, - get_options()->AccelDir)) { - log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting."); - return -1; - } - /* OP's don't need persistent keys; just make up an identity and * initialize the TLS context. */ if (!server_mode(options)) { - if (!(prkey = crypto_pk_new())) - return -1; - if (crypto_pk_generate_key(prkey)) { - crypto_pk_free(prkey); - return -1; - } - set_client_identity_key(prkey); - /* Create a TLS context. */ - if (router_initialize_tls_context() < 0) { - log_err(LD_GENERAL,"Error creating TLS context for Tor client."); - return -1; - } - return 0; + return init_keys_client(); } + if (init_keys_common() < 0) + return -1; /* Make sure DataDirectory exists, and is private. */ if (check_private_dir(options->DataDirectory, CPD_CREATE, options->User)) { return -1; diff --git a/src/or/router.h b/src/or/router.h index 61b35d6b5a..d8fcf0a9ad 100644 --- a/src/or/router.h +++ b/src/or/router.h @@ -37,6 +37,7 @@ void ntor_key_map_free(di_digest256_map_t *map); int router_initialize_tls_context(void); int init_keys(void); +int init_keys_client(void); int check_whether_orport_reachable(void); int check_whether_dirport_reachable(void); diff --git a/src/or/routerlist.c b/src/or/routerlist.c index dc48862201..8bd80391bc 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -3295,6 +3295,8 @@ router_add_to_routerlist(routerinfo_t *router, const char **msg, /* Make sure that it isn't expired. */ if (router->cert_expiration_time < approx_time()) { + routerinfo_free(router); + *msg = "Some certs on this router are expired."; return ROUTER_CERTS_EXPIRED; } @@ -4011,12 +4013,10 @@ update_all_descriptor_downloads(time_t now) void routerlist_retry_directory_downloads(time_t now) { + (void)now; router_reset_status_download_failures(); router_reset_descriptor_download_failures(); - if (get_options()->DisableNetwork) - return; - update_networkstatus_downloads(now); - update_all_descriptor_downloads(now); + reschedule_directory_downloads(); } /** Return true iff <b>router</b> does not permit exit streams. diff --git a/src/test/test.h b/src/test/test.h index b0c0946ac4..86699c3d07 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -40,6 +40,15 @@ tt_assert_test_type(a,b,#a" "#op" "#b,double,(val1_ op val2_),"%g", \ TT_EXIT_TEST_FUNCTION) +/* Declare "double equal" in a sneaky way, so compiler won't complain about + * comparing floats with == or !=. Of course, only do this if you know what + * you're doing. */ +#define tt_double_eq(a,b) \ + STMT_BEGIN \ + tt_double_op((a), >=, (b)); \ + tt_double_op((a), <=, (b)); \ + STMT_END + #ifdef _MSC_VER #define U64_PRINTF_TYPE uint64_t #define I64_PRINTF_TYPE int64_t diff --git a/src/test/test_containers.c b/src/test/test_containers.c index 2ae81bf18d..1ee240fb0d 100644 --- a/src/test/test_containers.c +++ b/src/test/test_containers.c @@ -887,7 +887,7 @@ static void test_container_order_functions(void *arg) { int lst[25], n = 0; - unsigned int lst_2[25]; + uint32_t lst_2[25]; // int a=12,b=24,c=25,d=60,e=77; #define median() median_int(lst, n) @@ -933,6 +933,31 @@ test_container_order_functions(void *arg) #undef third_quartile + double dbls[] = { 1.0, 10.0, 100.0, 1e4, 1e5, 1e6 }; + tt_double_eq(1.0, median_double(dbls, 1)); + tt_double_eq(1.0, median_double(dbls, 2)); + tt_double_eq(10.0, median_double(dbls, 3)); + tt_double_eq(10.0, median_double(dbls, 4)); + tt_double_eq(100.0, median_double(dbls, 5)); + tt_double_eq(100.0, median_double(dbls, 6)); + + time_t times[] = { 5, 10, 20, 25, 15 }; + + tt_assert(5 == median_time(times, 1)); + tt_assert(5 == median_time(times, 2)); + tt_assert(10 == median_time(times, 3)); + tt_assert(10 == median_time(times, 4)); + tt_assert(15 == median_time(times, 5)); + + int32_t int32s[] = { -5, -10, -50, 100 }; + tt_int_op(-5, ==, median_int32(int32s, 1)); + tt_int_op(-10, ==, median_int32(int32s, 2)); + tt_int_op(-10, ==, median_int32(int32s, 3)); + tt_int_op(-10, ==, median_int32(int32s, 4)); + + long longs[] = { -30, 30, 100, -100, 7 }; + tt_int_op(7, ==, find_nth_long(longs, 5, 2)); + done: ; } @@ -1078,6 +1103,129 @@ test_container_fp_pair_map(void *arg) tor_free(v105); } +static void +test_container_smartlist_most_frequent(void *arg) +{ + (void) arg; + smartlist_t *sl = smartlist_new(); + + int count = -1; + const char *cp; + + cp = smartlist_get_most_frequent_string_(sl, &count); + tt_int_op(count, ==, 0); + tt_ptr_op(cp, ==, NULL); + + /* String must be sorted before we call get_most_frequent */ + smartlist_split_string(sl, "abc:def:ghi", ":", 0, 0); + + cp = smartlist_get_most_frequent_string_(sl, &count); + tt_int_op(count, ==, 1); + tt_str_op(cp, ==, "ghi"); /* Ties broken in favor of later element */ + + smartlist_split_string(sl, "def:ghi", ":", 0, 0); + smartlist_sort_strings(sl); + + cp = smartlist_get_most_frequent_string_(sl, &count); + tt_int_op(count, ==, 2); + tt_ptr_op(cp, !=, NULL); + tt_str_op(cp, ==, "ghi"); /* Ties broken in favor of later element */ + + smartlist_split_string(sl, "def:abc:qwop", ":", 0, 0); + smartlist_sort_strings(sl); + + cp = smartlist_get_most_frequent_string_(sl, &count); + tt_int_op(count, ==, 3); + tt_ptr_op(cp, !=, NULL); + tt_str_op(cp, ==, "def"); /* No tie */ + + done: + SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp)); + smartlist_free(sl); +} + +static void +test_container_smartlist_sort_ptrs(void *arg) +{ + (void)arg; + int array[10]; + int *arrayptrs[11]; + smartlist_t *sl = smartlist_new(); + unsigned i=0, j; + + for (j = 0; j < ARRAY_LENGTH(array); ++j) { + smartlist_add(sl, &array[j]); + arrayptrs[i++] = &array[j]; + if (j == 5) { + smartlist_add(sl, &array[j]); + arrayptrs[i++] = &array[j]; + } + } + + for (i = 0; i < 10; ++i) { + smartlist_shuffle(sl); + smartlist_sort_pointers(sl); + for (j = 0; j < ARRAY_LENGTH(arrayptrs); ++j) { + tt_ptr_op(smartlist_get(sl, j), ==, arrayptrs[j]); + } + } + + done: + smartlist_free(sl); +} + +static void +test_container_smartlist_strings_eq(void *arg) +{ + (void)arg; + smartlist_t *sl1 = smartlist_new(); + smartlist_t *sl2 = smartlist_new(); +#define EQ_SHOULD_SAY(s1,s2,val) \ + do { \ + SMARTLIST_FOREACH(sl1, char *, cp, tor_free(cp)); \ + SMARTLIST_FOREACH(sl2, char *, cp, tor_free(cp)); \ + smartlist_clear(sl1); \ + smartlist_clear(sl2); \ + smartlist_split_string(sl1, (s1), ":", 0, 0); \ + smartlist_split_string(sl2, (s2), ":", 0, 0); \ + tt_int_op((val), OP_EQ, smartlist_strings_eq(sl1, sl2)); \ + } while (0) + + /* Both NULL, so equal */ + tt_int_op(1, ==, smartlist_strings_eq(NULL, NULL)); + + /* One NULL, not equal. */ + tt_int_op(0, ==, smartlist_strings_eq(NULL, sl1)); + tt_int_op(0, ==, smartlist_strings_eq(sl1, NULL)); + + /* Both empty, both equal. */ + EQ_SHOULD_SAY("", "", 1); + + /* One empty, not equal */ + EQ_SHOULD_SAY("", "ab", 0); + EQ_SHOULD_SAY("", "xy:z", 0); + EQ_SHOULD_SAY("abc", "", 0); + EQ_SHOULD_SAY("abc:cd", "", 0); + + /* Different lengths, not equal. */ + EQ_SHOULD_SAY("hello:world", "hello", 0); + EQ_SHOULD_SAY("hello", "hello:friends", 0); + + /* Same lengths, not equal */ + EQ_SHOULD_SAY("Hello:world", "goodbye:world", 0); + EQ_SHOULD_SAY("Hello:world", "Hello:stars", 0); + + /* Actually equal */ + EQ_SHOULD_SAY("ABC", "ABC", 1); + EQ_SHOULD_SAY(" ab : cd : e", " ab : cd : e", 1); + + done: + SMARTLIST_FOREACH(sl1, char *, cp, tor_free(cp)); + SMARTLIST_FOREACH(sl2, char *, cp, tor_free(cp)); + smartlist_free(sl1); + smartlist_free(sl2); +} + #define CONTAINER_LEGACY(name) \ { #name, test_container_ ## name , 0, NULL, NULL } @@ -1099,6 +1247,9 @@ struct testcase_t container_tests[] = { CONTAINER_LEGACY(order_functions), CONTAINER(di_map, 0), CONTAINER_LEGACY(fp_pair_map), + CONTAINER(smartlist_most_frequent, 0), + CONTAINER(smartlist_sort_ptrs, 0), + CONTAINER(smartlist_strings_eq, 0), END_OF_TESTCASES }; diff --git a/src/test/test_util.c b/src/test/test_util.c index a6c423dcdf..8b4513d34c 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -3611,6 +3611,50 @@ test_util_di_ops(void *arg) ; } +static void +test_util_di_map(void *arg) +{ + (void)arg; + di_digest256_map_t *dimap = NULL; + uint8_t key1[] = "Robert Anton Wilson "; + uint8_t key2[] = "Martin Gardner, _Fads&fallacies"; + uint8_t key3[] = "Tom Lehrer, _Be Prepared_. "; + uint8_t key4[] = "Ursula Le Guin,_A Wizard of... "; + + char dflt_entry[] = "'You have made a good beginning', but no more"; + + tt_int_op(32, ==, sizeof(key1)); + tt_int_op(32, ==, sizeof(key2)); + tt_int_op(32, ==, sizeof(key3)); + + tt_ptr_op(dflt_entry, ==, dimap_search(dimap, key1, dflt_entry)); + + char *str1 = tor_strdup("You are precisely as big as what you love" + " and precisely as small as what you allow" + " to annoy you."); + char *str2 = tor_strdup("Let us hope that Lysenko's success in Russia will" + " serve for many generations to come as another" + " reminder to the world of how quickly and easily" + " a science can be corrupted when ignorant" + " political leaders deem themselves competent" + " to arbitrate scientific disputes"); + char *str3 = tor_strdup("Don't write naughty words on walls " + "if you can't spell."); + + dimap_add_entry(&dimap, key1, str1); + dimap_add_entry(&dimap, key2, str2); + dimap_add_entry(&dimap, key3, str3); + + tt_ptr_op(str1, ==, dimap_search(dimap, key1, dflt_entry)); + tt_ptr_op(str3, ==, dimap_search(dimap, key3, dflt_entry)); + tt_ptr_op(str2, ==, dimap_search(dimap, key2, dflt_entry)); + tt_ptr_op(dflt_entry, ==, dimap_search(dimap, key4, dflt_entry)); + + done: + dimap_free(dimap, tor_free_); +} + + /** * Test counting high bits */ @@ -4395,6 +4439,7 @@ struct testcase_t util_tests[] = { UTIL_LEGACY(path_is_relative), UTIL_LEGACY(strtok), UTIL_LEGACY(di_ops), + UTIL_TEST(di_map, 0), UTIL_TEST(round_to_next_multiple_of, 0), UTIL_TEST(laplace, 0), UTIL_TEST(find_str_at_start_of_line, 0), |