diff options
Diffstat (limited to 'src')
200 files changed, 17422 insertions, 1916 deletions
diff --git a/src/common/address.c b/src/common/address.c index 793a40effc..5cd7a8df67 100644 --- a/src/common/address.c +++ b/src/common/address.c @@ -131,7 +131,8 @@ tor_addr_to_sockaddr(const tor_addr_t *a, #endif sin6->sin6_family = AF_INET6; sin6->sin6_port = htons(port); - memcpy(&sin6->sin6_addr, tor_addr_to_in6(a), sizeof(struct in6_addr)); + memcpy(&sin6->sin6_addr, tor_addr_to_in6_assert(a), + sizeof(struct in6_addr)); return sizeof(struct sockaddr_in6); } else { return 0; @@ -334,7 +335,7 @@ tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr) } else if (ent->h_addrtype == AF_INET6) { tor_addr_from_in6(addr, (struct in6_addr*) ent->h_addr); } else { - tor_assert(0); /* gethostbyname() returned a bizarre addrtype */ + tor_assert(0); // LCOV_EXCL_LINE: gethostbyname() returned bizarre type } return 0; } @@ -905,8 +906,10 @@ tor_addr_is_loopback(const tor_addr_t *addr) case AF_UNSPEC: return 0; default: + /* LCOV_EXCL_START */ tor_fragile_assert(); return 0; + /* LCOV_EXCL_STOP */ } } @@ -1027,7 +1030,7 @@ tor_addr_copy_tight(tor_addr_t *dest, const tor_addr_t *src) case AF_UNSPEC: break; default: - tor_fragile_assert(); + tor_fragile_assert(); // LCOV_EXCL_LINE } } @@ -1096,6 +1099,7 @@ tor_addr_compare_masked(const tor_addr_t *addr1, const tor_addr_t *addr2, case AF_INET6: { if (mbits > 128) mbits = 128; + const uint8_t *a1 = tor_addr_to_in6_addr8(addr1); const uint8_t *a2 = tor_addr_to_in6_addr8(addr2); const int bytes = mbits >> 3; @@ -1111,8 +1115,10 @@ tor_addr_compare_masked(const tor_addr_t *addr1, const tor_addr_t *addr2, } } default: + /* LCOV_EXCL_START */ tor_fragile_assert(); return 0; + /* LCOV_EXCL_STOP */ } } else if (how == CMP_EXACT) { /* Unequal families and an exact comparison? Stop now! */ @@ -1165,14 +1171,16 @@ tor_addr_hash(const tor_addr_t *addr) case AF_INET6: return siphash24g(&addr->addr.in6_addr.s6_addr, 16); default: + /* LCOV_EXCL_START */ tor_fragile_assert(); return 0; + /* LCOV_EXCL_END */ } } /** Return a newly allocated string with a representation of <b>addr</b>. */ char * -tor_dup_addr(const tor_addr_t *addr) +tor_addr_to_str_dup(const tor_addr_t *addr) { char buf[TOR_ADDR_BUF_LEN]; if (tor_addr_to_str(buf, addr, sizeof(buf), 0)) { @@ -1809,7 +1817,7 @@ MOCK_IMPL(smartlist_t *,get_interface_address6_list,(int severity, /* ====== * IPv4 helpers - * XXXX024 IPv6 deprecate some of these. + * XXXX IPv6 deprecate some of these. */ /** Given an address of the form "ip:port", try to divide it into its diff --git a/src/common/address.h b/src/common/address.h index 53712bde02..51db42c315 100644 --- a/src/common/address.h +++ b/src/common/address.h @@ -74,6 +74,8 @@ typedef struct tor_addr_port_t #define TOR_ADDR_NULL {AF_UNSPEC, {0}} static inline const struct in6_addr *tor_addr_to_in6(const tor_addr_t *a); +static inline const struct in6_addr *tor_addr_to_in6_assert( + const tor_addr_t *a); static inline uint32_t tor_addr_to_ipv4n(const tor_addr_t *a); static inline uint32_t tor_addr_to_ipv4h(const tor_addr_t *a); static inline uint32_t tor_addr_to_mapped_ipv4h(const tor_addr_t *a); @@ -97,21 +99,31 @@ tor_addr_to_in6(const tor_addr_t *a) return a->family == AF_INET6 ? &a->addr.in6_addr : NULL; } +/** As tor_addr_to_in6, but assert that the address truly is an IPv6 + * address. */ +static inline const struct in6_addr * +tor_addr_to_in6_assert(const tor_addr_t *a) +{ + tor_assert(a->family == AF_INET6); + return &a->addr.in6_addr; +} + /** Given an IPv6 address <b>x</b>, yield it as an array of uint8_t. * * Requires that <b>x</b> is actually an IPv6 address. */ -#define tor_addr_to_in6_addr8(x) tor_addr_to_in6(x)->s6_addr +#define tor_addr_to_in6_addr8(x) tor_addr_to_in6_assert(x)->s6_addr + /** Given an IPv6 address <b>x</b>, yield it as an array of uint16_t. * * Requires that <b>x</b> is actually an IPv6 address. */ -#define tor_addr_to_in6_addr16(x) S6_ADDR16(*tor_addr_to_in6(x)) +#define tor_addr_to_in6_addr16(x) S6_ADDR16(*tor_addr_to_in6_assert(x)) /** Given an IPv6 address <b>x</b>, yield it as an array of uint32_t. * * Requires that <b>x</b> is actually an IPv6 address. */ -#define tor_addr_to_in6_addr32(x) S6_ADDR32(*tor_addr_to_in6(x)) +#define tor_addr_to_in6_addr32(x) S6_ADDR32(*tor_addr_to_in6_assert(x)) /** Return an IPv4 address in network order for <b>a</b>, or 0 if * <b>a</b> is not an IPv4 address. */ @@ -179,7 +191,7 @@ tor_addr_eq_ipv4h(const tor_addr_t *a, uint32_t u) #define TOR_ADDR_BUF_LEN 48 int tor_addr_lookup(const char *name, uint16_t family, tor_addr_t *addr_out); -char *tor_dup_addr(const tor_addr_t *addr) ATTR_MALLOC; +char *tor_addr_to_str_dup(const tor_addr_t *addr) ATTR_MALLOC; /** Wrapper function of fmt_addr_impl(). It does not decorate IPv6 * addresses. */ diff --git a/src/common/aes.c b/src/common/aes.c index 15970a73f0..2b8a68c4a2 100644 --- a/src/common/aes.c +++ b/src/common/aes.c @@ -23,18 +23,7 @@ #error "We require OpenSSL >= 1.0.0" #endif -#ifdef __GNUC__ -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -#endif - -#if __GNUC__ && GCC_VERSION >= 402 -#if GCC_VERSION >= 406 -#pragma GCC diagnostic push -#endif -/* Some versions of OpenSSL declare SSL_get_selected_srtp_profile twice in - * srtp.h. Suppress the GCC warning so we can build with -Wredundant-decl. */ -#pragma GCC diagnostic ignored "-Wredundant-decls" -#endif +DISABLE_GCC_WARNING(redundant-decls) #include <assert.h> #include <stdlib.h> @@ -44,13 +33,7 @@ #include <openssl/engine.h> #include <openssl/modes.h> -#if __GNUC__ && GCC_VERSION >= 402 -#if GCC_VERSION >= 406 -#pragma GCC diagnostic pop -#else -#pragma GCC diagnostic warning "-Wredundant-decls" -#endif -#endif +ENABLE_GCC_WARNING(redundant-decls) #include "compat.h" #include "aes.h" @@ -255,9 +238,11 @@ evaluate_ctr_for_aes(void) if (fast_memneq(output, encrypt_zero, 16)) { /* Counter mode is buggy */ + /* LCOV_EXCL_START */ log_err(LD_CRYPTO, "This OpenSSL has a buggy version of counter mode; " "quitting tor."); exit(1); + /* LCOV_EXCL_STOP */ } return 0; } @@ -300,7 +285,7 @@ aes_set_key(aes_cnt_cipher_t *cipher, const char *key, int key_bits) case 128: c = EVP_aes_128_ecb(); break; case 192: c = EVP_aes_192_ecb(); break; case 256: c = EVP_aes_256_ecb(); break; - default: tor_assert(0); + default: tor_assert(0); // LCOV_EXCL_LINE } EVP_EncryptInit(&cipher->key.evp, c, (const unsigned char*)key, NULL); cipher->using_evp = 1; diff --git a/src/common/backtrace.c b/src/common/backtrace.c index 3b762b68e3..2841281927 100644 --- a/src/common/backtrace.c +++ b/src/common/backtrace.c @@ -13,9 +13,6 @@ * detect crashes. */ -#define __USE_GNU -#define _GNU_SOURCE 1 - #include "orconfig.h" #include "compat.h" #include "util.h" @@ -112,8 +109,10 @@ log_backtrace(int severity, int domain, const char *msg) tor_log(severity, domain, "%s. Stack trace:", msg); if (!symbols) { + /* LCOV_EXCL_START -- we can't provoke this. */ tor_log(severity, domain, " Unable to generate backtrace."); goto done; + /* LCOV_EXCL_STOP */ } for (i=0; i < depth; ++i) { tor_log(severity, domain, " %s", symbols[i]); @@ -176,8 +175,10 @@ install_bt_handler(void) for (i = 0; trap_signals[i] >= 0; ++i) { if (sigaction(trap_signals[i], &sa, NULL) == -1) { + /* LCOV_EXCL_START */ log_warn(LD_BUG, "Sigaction failed: %s", strerror(errno)); rv = -1; + /* LCOV_EXCL_STOP */ } } diff --git a/src/common/compat.c b/src/common/compat.c index 23eaa134cf..bae76f45db 100644 --- a/src/common/compat.c +++ b/src/common/compat.c @@ -12,17 +12,6 @@ * the platform. **/ -/* This is required on rh7 to make strptime not complain. - * We also need it to make memmem get defined (where available) - */ -/* XXXX024 We should just use AC_USE_SYSTEM_EXTENSIONS in our autoconf, - * and get this (and other important stuff!) automatically. Once we do that, - * make sure to also change the extern char **environ detection in - * configure.ac, because whether that is declared or not depends on whether - * we have _GNU_SOURCE defined! Maybe that means that once we take this out, - * we can also take out the configure check. */ -#define _GNU_SOURCE - #define COMPAT_PRIVATE #include "compat.h" @@ -525,8 +514,10 @@ tor_asprintf(char **strp, const char *fmt, ...) r = tor_vasprintf(strp, fmt, args); va_end(args); if (!*strp || r < 0) { + /* LCOV_EXCL_START */ log_err(LD_BUG, "Internal error in asprintf"); tor_assert(0); + /* LCOV_EXCL_STOP */ } return r; } @@ -1154,14 +1145,12 @@ tor_close_socket(tor_socket_t s) --n_sockets_open; #else if (r != EBADF) - --n_sockets_open; + --n_sockets_open; // LCOV_EXCL_LINE -- EIO and EINTR too hard to force. #endif r = -1; } - if (n_sockets_open < 0) - log_warn(LD_BUG, "Our socket count is below zero: %d. Please submit a " - "bug report.", n_sockets_open); + tor_assert_nonfatal(n_sockets_open >= 0); socket_accounting_unlock(); return r; } @@ -1941,7 +1930,7 @@ tor_getpwnam(const char *username) return NULL; if (! strcmp(username, passwd_cached->pw_name)) - return passwd_cached; + return passwd_cached; // LCOV_EXCL_LINE - would need to make getpwnam flaky return NULL; } @@ -1967,7 +1956,7 @@ tor_getpwuid(uid_t uid) return NULL; if (uid == passwd_cached->pw_uid) - return passwd_cached; + return passwd_cached; // LCOV_EXCL_LINE - would need to make getpwnam flaky return NULL; } @@ -2350,28 +2339,15 @@ get_parent_directory(char *fname) static char * alloc_getcwd(void) { - int saved_errno = errno; -/* We use this as a starting path length. Not too large seems sane. */ -#define START_PATH_LENGTH 128 -/* Nobody has a maxpath longer than this, as far as I know. And if they - * do, they shouldn't. */ -#define MAX_SANE_PATH_LENGTH 4096 - size_t path_length = START_PATH_LENGTH; - char *path = tor_malloc(path_length); - - errno = 0; - while (getcwd(path, path_length) == NULL) { - if (errno == ERANGE && path_length < MAX_SANE_PATH_LENGTH) { - path_length*=2; - path = tor_realloc(path, path_length); - } else { - tor_free(path); - path = NULL; - break; - } - } - errno = saved_errno; - return path; +#ifdef PATH_MAX +#define MAX_CWD PATH_MAX +#else +#define MAX_CWD 4096 +#endif + + char path_buf[MAX_CWD]; + char *path = getcwd(path_buf, sizeof(path_buf)); + return path ? tor_strdup(path) : NULL; } #endif @@ -2402,11 +2378,13 @@ make_path_absolute(char *fname) tor_asprintf(&absfname, "%s/%s", path, fname); tor_free(path); } else { + /* LCOV_EXCL_START Can't make getcwd fail. */ /* If getcwd failed, the best we can do here is keep using the * relative path. (Perhaps / isn't readable by this UID/GID.) */ log_warn(LD_GENERAL, "Unable to find current working directory: %s", strerror(errno)); absfname = tor_strdup(fname); + /* LCOV_EXCL_STOP */ } } return absfname; @@ -2770,7 +2748,9 @@ MOCK_IMPL(const char *, get_uname, (void)) } #endif #else + /* LCOV_EXCL_START -- can't provoke uname failure */ strlcpy(uname_result, "Unknown platform", sizeof(uname_result)); + /* LCOV_EXCL_STOP */ #endif } uname_result_is_set = 1; @@ -2844,11 +2824,14 @@ compute_num_cpus(void) if (num_cpus == -2) { num_cpus = compute_num_cpus_impl(); tor_assert(num_cpus != -2); - if (num_cpus > MAX_DETECTABLE_CPUS) + if (num_cpus > MAX_DETECTABLE_CPUS) { + /* LCOV_EXCL_START */ log_notice(LD_GENERAL, "Wow! I detected that you have %d CPUs. I " "will not autodetect any more than %d, though. If you " "want to configure more, set NumCPUs in your torrc", num_cpus, MAX_DETECTABLE_CPUS); + /* LCOV_EXCL_STOP */ + } } return num_cpus; } @@ -2873,18 +2856,22 @@ tor_gettimeofday(struct timeval *timeval) /* number of 100-nsec units since Jan 1, 1601 */ GetSystemTimeAsFileTime(&ft.ft_ft); if (ft.ft_64 < EPOCH_BIAS) { + /* LCOV_EXCL_START */ log_err(LD_GENERAL,"System time is before 1970; failing."); exit(1); + /* LCOV_EXCL_STOP */ } ft.ft_64 -= EPOCH_BIAS; timeval->tv_sec = (unsigned) (ft.ft_64 / UNITS_PER_SEC); timeval->tv_usec = (unsigned) ((ft.ft_64 / UNITS_PER_USEC) % USEC_PER_SEC); #elif defined(HAVE_GETTIMEOFDAY) if (gettimeofday(timeval, NULL)) { + /* LCOV_EXCL_START */ log_err(LD_GENERAL,"gettimeofday failed."); /* If gettimeofday dies, we have either given a bad timezone (we didn't), or segfaulted.*/ exit(1); + /* LCOV_EXCL_STOP */ } #elif defined(HAVE_FTIME) struct timeb tb; @@ -2975,11 +2962,12 @@ correct_tm(int islocal, const time_t *timep, struct tm *resultbuf, /* If we get here, then gmtime/localtime failed without getting an extreme * value for *timep */ - + /* LCOV_EXCL_START */ tor_fragile_assert(); r = resultbuf; memset(resultbuf, 0, sizeof(struct tm)); outcome="can't recover"; + /* LCOV_EXCL_STOP */ done: log_warn(LD_BUG, "%s("I64_FORMAT") failed with error %s: %s", islocal?"localtime":"gmtime", @@ -3373,9 +3361,11 @@ get_total_system_memory_impl(void) return result * 1024; err: + /* LCOV_EXCL_START Can't reach this unless proc is broken. */ tor_free(s); close(fd); return 0; + /* LCOV_EXCL_STOP */ #elif defined (_WIN32) /* Windows has MEMORYSTATUSEX; pretty straightforward. */ MEMORYSTATUSEX ms; @@ -3424,6 +3414,7 @@ get_total_system_memory(size_t *mem_out) static size_t mem_cached=0; uint64_t m = get_total_system_memory_impl(); if (0 == m) { + /* LCOV_EXCL_START -- can't make this happen without mocking. */ /* We couldn't find our memory total */ if (0 == mem_cached) { /* We have no cached value either */ @@ -3433,6 +3424,7 @@ get_total_system_memory(size_t *mem_out) *mem_out = mem_cached; return 0; + /* LCOV_EXCL_STOP */ } #if SIZE_MAX != UINT64_MAX diff --git a/src/common/compat.h b/src/common/compat.h index 8cf84580c6..6f102becc2 100644 --- a/src/common/compat.h +++ b/src/common/compat.h @@ -82,6 +82,44 @@ #define CHECK_SCANF(formatIdx, firstArg) #endif +/* What GCC do we have? */ +#ifdef __GNUC__ +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +#else +#define GCC_VERSION 0 +#endif + +/* Temporarily enable and disable warnings. */ +#ifdef __GNUC__ +# define PRAGMA_STRINGIFY_(s) #s +# define PRAGMA_JOIN_STRINGIFY_(a,b) PRAGMA_STRINGIFY_(a ## b) +/* Support for macro-generated pragmas (c99) */ +# define PRAGMA_(x) _Pragma (#x) +# ifdef __clang__ +# define PRAGMA_DIAGNOSTIC_(x) PRAGMA_(clang diagnostic x) +# else +# define PRAGMA_DIAGNOSTIC_(x) PRAGMA_(GCC diagnostic x) +# endif +# if defined(__clang__) || GCC_VERSION >= 406 +/* we have push/pop support */ +# define DISABLE_GCC_WARNING(warning) \ + PRAGMA_DIAGNOSTIC_(push) \ + PRAGMA_DIAGNOSTIC_(ignored PRAGMA_JOIN_STRINGIFY_(-W,warning)) +# define ENABLE_GCC_WARNING(warning) \ + PRAGMA_DIAGNOSTIC_(pop) +# else +/* older version of gcc: no push/pop support. */ +# define DISABLE_GCC_WARNING(warning) \ + PRAGMA_DIAGNOSTIC_(ignored PRAGMA_JOIN_STRINGIFY_(-W,warning)) +# define ENABLE_GCC_WARNING(warning) \ + PRAGMA_DIAGNOSTIC_(warning PRAGMA_JOIN_STRINGIFY_(-W,warning)) +# endif +#else /* ifdef __GNUC__ */ +/* not gcc at all */ +# define DISABLE_GCC_WARNING(warning) +# define ENABLE_GCC_WARNING(warning) +#endif + /* inline is __inline on windows. */ #ifdef _WIN32 #define inline __inline @@ -430,7 +468,7 @@ typedef int socklen_t; #ifdef _WIN32 /* XXX Actually, this should arguably be SOCKET; we use intptr_t here so that - * any inadvertant checks for the socket being <= 0 or > 0 will probably + * any inadvertent checks for the socket being <= 0 or > 0 will probably * still work. */ #define tor_socket_t intptr_t #define TOR_SOCKET_T_FORMAT INTPTR_T_FORMAT diff --git a/src/common/compat_libevent.c b/src/common/compat_libevent.c index cc58883750..4469f15ac7 100644 --- a/src/common/compat_libevent.c +++ b/src/common/compat_libevent.c @@ -125,7 +125,7 @@ tor_event_free(struct event *ev) #endif /** Global event base for use by the main thread. */ -struct event_base *the_event_base = NULL; +static struct event_base *the_event_base = NULL; /* This is what passes for version detection on OSX. We set * MACOSX_KQUEUE_IS_BROKEN to true iff we're on a version of OSX before @@ -228,8 +228,10 @@ tor_libevent_initialize(tor_libevent_cfg *torcfg) #endif if (!the_event_base) { + /* LCOV_EXCL_START */ log_err(LD_GENERAL, "Unable to initialize Libevent: cannot continue."); exit(1); + /* LCOV_EXCL_STOP */ } /* Making this a NOTICE for now so we can link bugs to a libevent versions diff --git a/src/common/compat_pthreads.c b/src/common/compat_pthreads.c index 1b24cc3c2a..79f5cec43a 100644 --- a/src/common/compat_pthreads.c +++ b/src/common/compat_pthreads.c @@ -10,8 +10,6 @@ * functions. */ -#define _GNU_SOURCE - #include "orconfig.h" #include <pthread.h> #include <signal.h> @@ -104,11 +102,13 @@ void tor_mutex_init(tor_mutex_t *mutex) { if (PREDICT_UNLIKELY(!threads_initialized)) - tor_threads_init(); + tor_threads_init(); // LCOV_EXCL_LINE const int err = pthread_mutex_init(&mutex->mutex, &attr_recursive); if (PREDICT_UNLIKELY(err)) { + // LCOV_EXCL_START log_err(LD_GENERAL, "Error %d creating a mutex.", err); - tor_fragile_assert(); + tor_assert_unreached(); + // LCOV_EXCL_STOP } } @@ -118,12 +118,14 @@ void tor_mutex_init_nonrecursive(tor_mutex_t *mutex) { int err; - if (PREDICT_UNLIKELY(!threads_initialized)) - tor_threads_init(); + if (!threads_initialized) + tor_threads_init(); // LCOV_EXCL_LINE err = pthread_mutex_init(&mutex->mutex, NULL); if (PREDICT_UNLIKELY(err)) { + // LCOV_EXCL_START log_err(LD_GENERAL, "Error %d creating a mutex.", err); - tor_fragile_assert(); + tor_assert_unreached(); + // LCOV_EXCL_STOP } } @@ -135,8 +137,10 @@ tor_mutex_acquire(tor_mutex_t *m) tor_assert(m); err = pthread_mutex_lock(&m->mutex); if (PREDICT_UNLIKELY(err)) { + // LCOV_EXCL_START log_err(LD_GENERAL, "Error %d locking a mutex.", err); - tor_fragile_assert(); + tor_assert_unreached(); + // LCOV_EXCL_STOP } } /** Release the lock <b>m</b> so another thread can have it. */ @@ -147,8 +151,10 @@ tor_mutex_release(tor_mutex_t *m) tor_assert(m); err = pthread_mutex_unlock(&m->mutex); if (PREDICT_UNLIKELY(err)) { + // LCOV_EXCL_START log_err(LD_GENERAL, "Error %d unlocking a mutex.", err); - tor_fragile_assert(); + tor_assert_unreached(); + // LCOV_EXCL_STOP } } /** Clean up the mutex <b>m</b> so that it no longer uses any system @@ -161,8 +167,10 @@ tor_mutex_uninit(tor_mutex_t *m) tor_assert(m); err = pthread_mutex_destroy(&m->mutex); if (PREDICT_UNLIKELY(err)) { + // LCOV_EXCL_START log_err(LD_GENERAL, "Error %d destroying a mutex.", err); - tor_fragile_assert(); + tor_assert_unreached(); + // LCOV_EXCL_STOP } } /** Return an integer representing this thread. */ @@ -212,8 +220,10 @@ void tor_cond_uninit(tor_cond_t *cond) { if (pthread_cond_destroy(&cond->cond)) { + // LCOV_EXCL_START log_warn(LD_GENERAL,"Error freeing condition: %s", strerror(errno)); return; + // LCOV_EXCL_STOP } } /** Wait until one of the tor_cond_signal functions is called on <b>cond</b>. @@ -234,7 +244,7 @@ tor_cond_wait(tor_cond_t *cond, tor_mutex_t *mutex, const struct timeval *tv) /* EINTR should be impossible according to POSIX, but POSIX, like the * Pirate's Code, is apparently treated "more like what you'd call * guidelines than actual rules." */ - continue; + continue; // LCOV_EXCL_LINE } return r ? -1 : 0; } diff --git a/src/common/compat_threads.c b/src/common/compat_threads.c index 8f9001258a..f4809060d6 100644 --- a/src/common/compat_threads.c +++ b/src/common/compat_threads.c @@ -11,8 +11,6 @@ * modules.) */ -#define _GNU_SOURCE - #include "orconfig.h" #include <stdlib.h> #include "compat.h" @@ -63,8 +61,8 @@ tor_cond_t * tor_cond_new(void) { tor_cond_t *cond = tor_malloc(sizeof(tor_cond_t)); - if (tor_cond_init(cond)<0) - tor_free(cond); + if (BUG(tor_cond_init(cond)<0)) + tor_free(cond); // LCOV_EXCL_LINE return cond; } @@ -242,8 +240,11 @@ alert_sockets_create(alert_sockets_t *socks_out, uint32_t flags) if (socks[0] >= 0) { if (fcntl(socks[0], F_SETFD, FD_CLOEXEC) < 0 || set_socket_nonblocking(socks[0]) < 0) { + // LCOV_EXCL_START -- if eventfd succeeds, fcntl will. + tor_assert_nonfatal_unreached(); close(socks[0]); return -1; + // LCOV_EXCL_STOP } } } @@ -277,9 +278,12 @@ alert_sockets_create(alert_sockets_t *socks_out, uint32_t flags) fcntl(socks[1], F_SETFD, FD_CLOEXEC) < 0 || set_socket_nonblocking(socks[0]) < 0 || set_socket_nonblocking(socks[1]) < 0) { + // LCOV_EXCL_START -- if pipe succeeds, you can fcntl the output + tor_assert_nonfatal_unreached(); close(socks[0]); close(socks[1]); return -1; + // LCOV_EXCL_STOP } socks_out->read_fd = socks[0]; socks_out->write_fd = socks[1]; @@ -294,9 +298,12 @@ alert_sockets_create(alert_sockets_t *socks_out, uint32_t flags) tor_socketpair(AF_UNIX, SOCK_STREAM, 0, socks) == 0) { if (set_socket_nonblocking(socks[0]) < 0 || set_socket_nonblocking(socks[1])) { + // LCOV_EXCL_START -- if socketpair worked, you can make it nonblocking. + tor_assert_nonfatal_unreached(); tor_close_socket(socks[0]); tor_close_socket(socks[1]); return -1; + // LCOV_EXCL_STOP } socks_out->read_fd = socks[0]; socks_out->write_fd = socks[1]; diff --git a/src/common/crypto.c b/src/common/crypto.c index 2b96324d33..1c5b5993c9 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -29,18 +29,7 @@ #include "crypto_ed25519.h" #include "crypto_format.h" -#ifdef __GNUC__ -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -#endif - -#if __GNUC__ && GCC_VERSION >= 402 -#if GCC_VERSION >= 406 -#pragma GCC diagnostic push -#endif -/* Some versions of OpenSSL declare X509_STORE_CTX_set_verify_cb twice. - * Suppress the GCC warning so we can build with -Wredundant-decl. */ -#pragma GCC diagnostic ignored "-Wredundant-decls" -#endif +DISABLE_GCC_WARNING(redundant-decls) #include <openssl/err.h> #include <openssl/rsa.h> @@ -53,6 +42,8 @@ #include <openssl/conf.h> #include <openssl/hmac.h> +ENABLE_GCC_WARNING(redundant-decls) + #if __GNUC__ && GCC_VERSION >= 402 #if GCC_VERSION >= 406 #pragma GCC diagnostic pop @@ -65,7 +56,6 @@ #include <ctype.h> #endif #ifdef HAVE_UNISTD_H -#define _GNU_SOURCE #include <unistd.h> #endif #ifdef HAVE_FCNTL_H @@ -155,7 +145,7 @@ crypto_get_rsa_padding_overhead(int padding) switch (padding) { case RSA_PKCS1_OAEP_PADDING: return PKCS1_OAEP_PADDING_OVERHEAD; - default: tor_assert(0); return -1; + default: tor_assert(0); return -1; // LCOV_EXCL_LINE } } @@ -167,7 +157,7 @@ crypto_get_rsa_padding(int padding) switch (padding) { case PK_PKCS1_OAEP_PADDING: return RSA_PKCS1_OAEP_PADDING; - default: tor_assert(0); return -1; + default: tor_assert(0); return -1; // LCOV_EXCL_LINE } } @@ -192,13 +182,9 @@ crypto_log_errors(int severity, const char *doing) if (!msg) msg = "(null)"; if (!lib) lib = "(null)"; if (!func) func = "(null)"; - if (doing) { - tor_log(severity, LD_CRYPTO, "crypto error while %s: %s (in %s:%s)", + if (BUG(!doing)) doing = "(null)"; + tor_log(severity, LD_CRYPTO, "crypto error while %s: %s (in %s:%s)", doing, msg, lib, func); - } else { - tor_log(severity, LD_CRYPTO, "crypto error: %s (in %s:%s)", - msg, lib, func); - } } } @@ -1015,6 +1001,10 @@ crypto_pk_copy_full(crypto_pk_t *env) new_key = RSAPublicKey_dup(env->key); } if (!new_key) { + /* LCOV_EXCL_START + * + * We can't cause RSA*Key_dup() to fail, so we can't really test this. + */ log_err(LD_CRYPTO, "Unable to duplicate a %s key: openssl failed.", privatekey?"private":"public"); crypto_log_errors(LOG_ERR, @@ -1022,6 +1012,7 @@ crypto_pk_copy_full(crypto_pk_t *env) "Duplicating a public key"); tor_fragile_assert(); return NULL; + /* LCOV_EXCL_STOP */ } return crypto_new_pk_from_rsa_(new_key); @@ -1772,8 +1763,10 @@ crypto_digest_algorithm_get_name(digest_algorithm_t alg) case DIGEST_SHA3_512: return "sha3-512"; default: + // LCOV_EXCL_START tor_fragile_assert(); return "??unknown_digest??"; + // LCOV_EXCL_STOP } } @@ -1797,7 +1790,7 @@ crypto_digest_algorithm_parse_name(const char *name) } /** Given an algorithm, return the digest length in bytes. */ -static inline size_t +size_t crypto_digest_algorithm_get_length(digest_algorithm_t alg) { switch (alg) { @@ -1812,8 +1805,8 @@ crypto_digest_algorithm_get_length(digest_algorithm_t alg) case DIGEST_SHA3_512: return DIGEST512_LEN; default: - tor_assert(0); - return 0; /* Unreachable */ + tor_assert(0); // LCOV_EXCL_LINE + return 0; /* Unreachable */ // LCOV_EXCL_LINE } } @@ -1856,23 +1849,53 @@ crypto_digest_alloc_bytes(digest_algorithm_t alg) case DIGEST_SHA3_512: return END_OF_FIELD(d.sha3); default: - tor_assert(0); - return 0; + tor_assert(0); // LCOV_EXCL_LINE + return 0; // LCOV_EXCL_LINE } #undef END_OF_FIELD #undef STRUCT_FIELD_SIZE } +/** + * Internal function: create and return a new digest object for 'algorithm'. + * Does not typecheck the algorithm. + */ +static crypto_digest_t * +crypto_digest_new_internal(digest_algorithm_t algorithm) +{ + crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm)); + r->algorithm = algorithm; + + switch (algorithm) + { + case DIGEST_SHA1: + SHA1_Init(&r->d.sha1); + break; + case DIGEST_SHA256: + SHA256_Init(&r->d.sha2); + break; + case DIGEST_SHA512: + SHA512_Init(&r->d.sha512); + break; + case DIGEST_SHA3_256: + keccak_digest_init(&r->d.sha3, 256); + break; + case DIGEST_SHA3_512: + keccak_digest_init(&r->d.sha3, 512); + break; + default: + tor_assert_unreached(); + } + + return r; +} + /** Allocate and return a new digest object to compute SHA1 digests. */ crypto_digest_t * crypto_digest_new(void) { - crypto_digest_t *r; - r = tor_malloc(crypto_digest_alloc_bytes(DIGEST_SHA1)); - SHA1_Init(&r->d.sha1); - r->algorithm = DIGEST_SHA1; - return r; + return crypto_digest_new_internal(DIGEST_SHA1); } /** Allocate and return a new digest object to compute 256-bit digests @@ -1880,15 +1903,8 @@ crypto_digest_new(void) crypto_digest_t * crypto_digest256_new(digest_algorithm_t algorithm) { - crypto_digest_t *r; tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); - r = tor_malloc(crypto_digest_alloc_bytes(algorithm)); - if (algorithm == DIGEST_SHA256) - SHA256_Init(&r->d.sha2); - else - keccak_digest_init(&r->d.sha3, 256); - r->algorithm = algorithm; - return r; + return crypto_digest_new_internal(algorithm); } /** Allocate and return a new digest object to compute 512-bit digests @@ -1896,15 +1912,8 @@ crypto_digest256_new(digest_algorithm_t algorithm) crypto_digest_t * crypto_digest512_new(digest_algorithm_t algorithm) { - crypto_digest_t *r; tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); - r = tor_malloc(crypto_digest_alloc_bytes(algorithm)); - if (algorithm == DIGEST_SHA512) - SHA512_Init(&r->d.sha512); - else - keccak_digest_init(&r->d.sha3, 512); - r->algorithm = algorithm; - return r; + return crypto_digest_new_internal(algorithm); } /** Deallocate a digest object. @@ -1947,8 +1956,10 @@ crypto_digest_add_bytes(crypto_digest_t *digest, const char *data, keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len); break; default: + /* LCOV_EXCL_START */ tor_fragile_assert(); break; + /* LCOV_EXCL_STOP */ } } @@ -1987,13 +1998,15 @@ crypto_digest_get_digest(crypto_digest_t *digest, case DIGEST_SHA512: SHA512_Final(r, &tmpenv.d.sha512); break; +//LCOV_EXCL_START case DIGEST_SHA3_256: /* FALLSTHROUGH */ case DIGEST_SHA3_512: - log_warn(LD_BUG, "Handling unexpected algorithm %d", digest->algorithm); - tor_assert(0); /* This is fatal, because it should never happen. */ default: - tor_assert(0); /* Unreachable. */ + log_warn(LD_BUG, "Handling unexpected algorithm %d", digest->algorithm); + /* This is fatal, because it should never happen. */ + tor_assert_unreached(); break; +//LCOV_EXCL_STOP } memcpy(out, r, out_len); memwipe(r, 0, sizeof(r)); @@ -2052,27 +2065,7 @@ crypto_digest_smartlist_prefix(char *digest_out, size_t len_out, const char *append, digest_algorithm_t alg) { - crypto_digest_t *d = NULL; - switch (alg) { - case DIGEST_SHA1: - d = crypto_digest_new(); - break; - case DIGEST_SHA256: /* FALLSTHROUGH */ - case DIGEST_SHA3_256: - d = crypto_digest256_new(alg); - break; - case DIGEST_SHA512: /* FALLSTHROUGH */ - case DIGEST_SHA3_512: - d = crypto_digest512_new(alg); - break; - default: - log_warn(LD_BUG, "Called with unknown algorithm %d", alg); - /* If fragile_assert is not enabled, wipe output and return - * without running any calculations */ - memwipe(digest_out, 0xff, len_out); - tor_fragile_assert(); - goto free; - } + crypto_digest_t *d = crypto_digest_new_internal(alg); if (prepend) crypto_digest_add_bytes(d, prepend, strlen(prepend)); SMARTLIST_FOREACH(lst, const char *, cp, @@ -2080,8 +2073,6 @@ crypto_digest_smartlist_prefix(char *digest_out, size_t len_out, if (append) crypto_digest_add_bytes(d, append, strlen(append)); crypto_digest_get_digest(d, digest_out, len_out); - - free: crypto_digest_free(d); } @@ -2250,9 +2241,14 @@ crypto_set_tls_dh_prime(void) int r; /* If the space is occupied, free the previous TLS DH prime */ - if (dh_param_p_tls) { + if (BUG(dh_param_p_tls)) { + /* LCOV_EXCL_START + * + * We shouldn't be calling this twice. + */ BN_clear_free(dh_param_p_tls); dh_param_p_tls = NULL; + /* LCOV_EXCL_STOP */ } tls_prime = BN_new(); @@ -2284,8 +2280,8 @@ init_dh_param(void) { BIGNUM *circuit_dh_prime; int r; - if (dh_param_p && dh_param_g) - return; + if (BUG(dh_param_p && dh_param_g)) + return; // LCOV_EXCL_LINE This function isn't supposed to be called twice. circuit_dh_prime = BN_new(); tor_assert(circuit_dh_prime); @@ -2375,10 +2371,13 @@ crypto_dh_new(int dh_type) return res; err: + /* LCOV_EXCL_START + * This error condition is only reached when an allocation fails */ crypto_log_errors(LOG_WARN, "creating DH object"); if (res->dh) DH_free(res->dh); /* frees p and g too */ tor_free(res); return NULL; + /* LCOV_EXCL_STOP */ } /** Return a copy of <b>dh</b>, sharing its internal state. */ @@ -2412,8 +2411,11 @@ crypto_dh_generate_public(crypto_dh_t *dh) again: #endif if (!DH_generate_key(dh->dh)) { + /* LCOV_EXCL_START + * To test this we would need some way to tell openssl to break DH. */ crypto_log_errors(LOG_WARN, "generating DH key"); return -1; + /* LCOV_EXCL_STOP */ } #ifdef OPENSSL_1_1_API /* OpenSSL 1.1.x doesn't appear to let you regenerate a DH key, without @@ -2429,6 +2431,8 @@ crypto_dh_generate_public(crypto_dh_t *dh) } #else if (tor_check_dh_key(LOG_WARN, dh->dh->pub_key)<0) { + /* LCOV_EXCL_START + * If this happens, then openssl's DH implementation is busted. */ log_warn(LD_CRYPTO, "Weird! Our own DH key was invalid. I guess once-in-" "the-universe chances really do happen. Trying again."); /* Free and clear the keys, so OpenSSL will actually try again. */ @@ -2436,6 +2440,7 @@ crypto_dh_generate_public(crypto_dh_t *dh) BN_clear_free(dh->dh->priv_key); dh->dh->pub_key = dh->dh->priv_key = NULL; goto again; + /* LCOV_EXCL_STOP */ } #endif return 0; @@ -2500,8 +2505,8 @@ tor_check_dh_key(int severity, const BIGNUM *bn) tor_assert(bn); x = BN_new(); tor_assert(x); - if (!dh_param_p) - init_dh_param(); + if (BUG(!dh_param_p)) + init_dh_param(); //LCOV_EXCL_LINE we already checked whether we did this. BN_set_word(x, 1); if (BN_cmp(bn,x)<=0) { log_fn(severity, LD_CRYPTO, "DH key must be at least 2."); @@ -2523,8 +2528,6 @@ tor_check_dh_key(int severity, const BIGNUM *bn) return -1; } -#undef MIN -#define MIN(a,b) ((a)<(b)?(a):(b)) /** Given a DH key exchange object, and our peer's value of g^y (as a * <b>pubkey_len</b>-byte value in <b>pubkey</b>) generate * <b>secret_bytes_out</b> bytes of shared key material and write them @@ -2712,6 +2715,11 @@ crypto_seed_weak_rng(tor_weak_rng_t *rng) tor_init_weak_random(rng, seed); } +#ifdef TOR_UNIT_TESTS +int break_strongest_rng_syscall = 0; +int break_strongest_rng_fallback = 0; +#endif + /** Try to get <b>out_len</b> bytes of the strongest entropy we can generate, * via system calls, storing it into <b>out</b>. Return 0 on success, -1 on * failure. A maximum request size of 256 bytes is imposed. @@ -2721,6 +2729,11 @@ crypto_strongest_rand_syscall(uint8_t *out, size_t out_len) { tor_assert(out_len <= MAX_STRONGEST_RAND_SIZE); +#ifdef TOR_UNIT_TESTS + if (break_strongest_rng_syscall) + return -1; +#endif + #if defined(_WIN32) static int provider_set = 0; static HCRYPTPROV provider; @@ -2770,6 +2783,7 @@ crypto_strongest_rand_syscall(uint8_t *out, size_t out_len) } while (ret == -1 && ((errno == EINTR) ||(errno == EAGAIN))); if (PREDICT_UNLIKELY(ret == -1)) { + /* LCOV_EXCL_START we can't actually make the syscall fail in testing. */ tor_assert(errno != EAGAIN); tor_assert(errno != EINTR); @@ -2777,6 +2791,7 @@ crypto_strongest_rand_syscall(uint8_t *out, size_t out_len) log_warn(LD_CRYPTO, "Can't get entropy from getrandom()."); getrandom_works = 0; /* Don't bother trying again. */ return -1; + /* LCOV_EXCL_STOP */ } tor_assert(ret == (long)out_len); @@ -2805,6 +2820,11 @@ crypto_strongest_rand_syscall(uint8_t *out, size_t out_len) static int crypto_strongest_rand_fallback(uint8_t *out, size_t out_len) { +#ifdef TOR_UNIT_TESTS + if (break_strongest_rng_fallback) + return -1; +#endif + #ifdef _WIN32 /* Windows exclusively uses crypto_strongest_rand_syscall(). */ (void)out; @@ -2825,10 +2845,13 @@ crypto_strongest_rand_fallback(uint8_t *out, size_t out_len) n = read_all(fd, (char*)out, out_len, 0); close(fd); if (n != out_len) { + /* LCOV_EXCL_START + * We can't make /dev/foorandom actually fail. */ log_warn(LD_CRYPTO, "Error reading from entropy source (read only %lu bytes).", (unsigned long)n); return -1; + /* LCOV_EXCL_STOP */ } return 0; @@ -2842,7 +2865,7 @@ crypto_strongest_rand_fallback(uint8_t *out, size_t out_len) * storing it into <b>out</b>. Return 0 on success, -1 on failure. A maximum * request size of 256 bytes is imposed. */ -static int +STATIC int crypto_strongest_rand_raw(uint8_t *out, size_t out_len) { static const size_t sanity_min_size = 16; @@ -2876,13 +2899,17 @@ crypto_strongest_rand_raw(uint8_t *out, size_t out_len) return 0; } - /* We tried max_attempts times to fill a buffer >= 128 bits long, + /* LCOV_EXCL_START + * + * We tried max_attempts times to fill a buffer >= 128 bits long, * and each time it returned all '0's. Either the system entropy * source is busted, or the user should go out and buy a ticket to * every lottery on the planet. */ log_warn(LD_CRYPTO, "Strong OS entropy returned all zero buffer."); + return -1; + /* LCOV_EXCL_STOP */ } /** Try to get <b>out_len</b> bytes of the strongest entropy we can generate, @@ -2901,10 +2928,12 @@ crypto_strongest_rand(uint8_t *out, size_t out_len) while (out_len) { crypto_rand((char*) inp, DLEN); if (crypto_strongest_rand_raw(inp+DLEN, DLEN) < 0) { + // LCOV_EXCL_START log_err(LD_CRYPTO, "Failed to load strong entropy when generating an " "important key. Exiting."); /* Die with an assertion so we get a stack trace. */ tor_assert(0); + // LCOV_EXCL_STOP } if (out_len >= DLEN) { SHA512(inp, sizeof(inp), out); @@ -2935,7 +2964,7 @@ crypto_seed_rng(void) * functions. If one succeeds, we'll accept the RNG as seeded. */ rand_poll_ok = RAND_poll(); if (rand_poll_ok == 0) - log_warn(LD_CRYPTO, "RAND_poll() failed."); + log_warn(LD_CRYPTO, "RAND_poll() failed."); // LCOV_EXCL_LINE load_entropy_ok = !crypto_strongest_rand_raw(buf, sizeof(buf)); if (load_entropy_ok) { diff --git a/src/common/crypto.h b/src/common/crypto.h index 682c4e3253..f8fb0daa81 100644 --- a/src/common/crypto.h +++ b/src/common/crypto.h @@ -233,6 +233,7 @@ void crypto_digest_smartlist(char *digest_out, size_t len_out, const struct smartlist_t *lst, const char *append, digest_algorithm_t alg); const char *crypto_digest_algorithm_get_name(digest_algorithm_t alg); +size_t crypto_digest_algorithm_get_length(digest_algorithm_t alg); int crypto_digest_algorithm_parse_name(const char *name); crypto_digest_t *crypto_digest_new(void); crypto_digest_t *crypto_digest256_new(digest_algorithm_t algorithm); @@ -317,6 +318,12 @@ void crypto_add_spaces_to_fp(char *out, size_t outlen, const char *in); #ifdef CRYPTO_PRIVATE STATIC int crypto_force_rand_ssleay(void); +STATIC int crypto_strongest_rand_raw(uint8_t *out, size_t out_len); + +#ifdef TOR_UNIT_TESTS +extern int break_strongest_rng_syscall; +extern int break_strongest_rng_fallback; +#endif #endif #endif diff --git a/src/common/crypto_curve25519.c b/src/common/crypto_curve25519.c index 57c878b79a..58ec923638 100644 --- a/src/common/crypto_curve25519.c +++ b/src/common/crypto_curve25519.c @@ -65,8 +65,10 @@ STATIC int curve25519_basepoint_impl(uint8_t *output, const uint8_t *secret) { int r = 0; - if (PREDICT_UNLIKELY(curve25519_use_ed == -1)) { + if (BUG(curve25519_use_ed == -1)) { + /* LCOV_EXCL_START - Only reached if we forgot to call curve25519_init() */ pick_curve25519_basepoint_impl(); + /* LCOV_EXCL_STOP */ } /* TODO: Someone should benchmark curved25519_scalarmult_basepoint versus @@ -290,10 +292,13 @@ pick_curve25519_basepoint_impl(void) if (curve25519_basepoint_spot_check() == 0) return; - log_warn(LD_CRYPTO, "The ed25519-based curve25519 basepoint " + /* LCOV_EXCL_START + * only reachable if our basepoint implementation broken */ + log_warn(LD_BUG|LD_CRYPTO, "The ed25519-based curve25519 basepoint " "multiplication seems broken; using the curve25519 " "implementation."); curve25519_use_ed = 0; + /* LCOV_EXCL_STOP */ } /** Initialize the curve25519 implementations. This is necessary if you're diff --git a/src/common/crypto_ed25519.c b/src/common/crypto_ed25519.c index ea2d8e3892..84c3eece6d 100644 --- a/src/common/crypto_ed25519.c +++ b/src/common/crypto_ed25519.c @@ -94,8 +94,8 @@ static const ed25519_impl_t *ed25519_impl = NULL; static inline const ed25519_impl_t * get_ed_impl(void) { - if (PREDICT_UNLIKELY(ed25519_impl == NULL)) { - pick_ed25519_impl(); + if (BUG(ed25519_impl == NULL)) { + pick_ed25519_impl(); // LCOV_EXCL_LINE - We always call ed25519_init(). } return ed25519_impl; } @@ -259,11 +259,11 @@ ed25519_checksig_batch(int *okay_out, int *oks; int all_ok; - ms = tor_malloc(sizeof(uint8_t*)*n_checkable); - lens = tor_malloc(sizeof(size_t)*n_checkable); - pks = tor_malloc(sizeof(uint8_t*)*n_checkable); - sigs = tor_malloc(sizeof(uint8_t*)*n_checkable); - oks = okay_out ? okay_out : tor_malloc(sizeof(int)*n_checkable); + ms = tor_calloc(n_checkable, sizeof(uint8_t*)); + lens = tor_calloc(n_checkable, sizeof(size_t)); + pks = tor_calloc(n_checkable, sizeof(uint8_t*)); + sigs = tor_calloc(n_checkable, sizeof(uint8_t*)); + oks = okay_out ? okay_out : tor_calloc(n_checkable, sizeof(int)); for (i = 0; i < n_checkable; ++i) { ms[i] = checkable[i].msg; @@ -433,6 +433,7 @@ ed25519_seckey_read_from_file(ed25519_secret_key_t *seckey_out, errno = EINVAL; } + tor_free(*tag_out); return -1; } @@ -472,6 +473,7 @@ ed25519_pubkey_read_from_file(ed25519_public_key_t *pubkey_out, errno = EINVAL; } + tor_free(*tag_out); return -1; } @@ -594,9 +596,12 @@ pick_ed25519_impl(void) if (ed25519_impl_spot_check() == 0) return; + /* LCOV_EXCL_START + * unreachable unless ed25519_donna is broken */ log_warn(LD_CRYPTO, "The Ed25519-donna implementation seems broken; using " "the ref10 implementation."); ed25519_impl = &impl_ref10; + /* LCOV_EXCL_STOP */ } /* Initialize the Ed25519 implementation. This is neccessary if you're diff --git a/src/common/crypto_pwbox.c b/src/common/crypto_pwbox.c index 819dc0c39d..31e37c007d 100644 --- a/src/common/crypto_pwbox.c +++ b/src/common/crypto_pwbox.c @@ -61,7 +61,7 @@ crypto_pwbox(uint8_t **out, size_t *outlen_out, pwbox_encoded_getarray_skey_header(enc), S2K_MAXLEN, s2k_flags); - if (spec_len < 0 || spec_len > S2K_MAXLEN) + if (BUG(spec_len < 0 || spec_len > S2K_MAXLEN)) goto err; pwbox_encoded_setlen_skey_header(enc, spec_len); enc->header_len = spec_len; @@ -76,10 +76,11 @@ crypto_pwbox(uint8_t **out, size_t *outlen_out, /* Now that all the data is in position, derive some keys, encrypt, and * digest */ - if (secret_to_key_derivekey(keys, sizeof(keys), + const int s2k_rv = secret_to_key_derivekey(keys, sizeof(keys), pwbox_encoded_getarray_skey_header(enc), spec_len, - secret, secret_len) < 0) + secret, secret_len); + if (BUG(s2k_rv < 0)) goto err; cipher = crypto_cipher_new_with_iv((char*)keys, (char*)enc->iv); @@ -87,11 +88,11 @@ crypto_pwbox(uint8_t **out, size_t *outlen_out, crypto_cipher_free(cipher); result_len = pwbox_encoded_encoded_len(enc); - if (result_len < 0) + if (BUG(result_len < 0)) goto err; result = tor_malloc(result_len); enc_len = pwbox_encoded_encode(result, result_len, enc); - if (enc_len < 0) + if (BUG(enc_len < 0)) goto err; tor_assert(enc_len == result_len); @@ -107,9 +108,24 @@ crypto_pwbox(uint8_t **out, size_t *outlen_out, goto out; err: + /* LCOV_EXCL_START + + This error case is often unreachable if we're correctly coded, unless + somebody adds a new error case somewhere, or unless you're building + without scrypto support. + + - make_specifier can't fail, unless S2K_MAX_LEN is too short. + - secret_to_key_derivekey can't really fail unless we're missing + scrypt, or the underlying function fails, or we pass it a bogus + algorithm or parameters. + - pwbox_encoded_encoded_len can't fail unless we're using trunnel + incorrectly. + - pwbox_encoded_encode can't fail unless we're using trunnel wrong, + or it's buggy. + */ tor_free(result); rv = -1; - + /* LCOV_EXCL_STOP */ out: pwbox_encoded_free(enc); memwipe(keys, 0, sizeof(keys)); diff --git a/src/common/crypto_s2k.c b/src/common/crypto_s2k.c index 3bc05f1cf9..5dbd2ad91f 100644 --- a/src/common/crypto_s2k.c +++ b/src/common/crypto_s2k.c @@ -57,7 +57,8 @@ #define SCRYPT_KEY_LEN 32 /** Given an algorithm ID (one of S2K_TYPE_*), return the length of the - * specifier part of it, without the prefix type byte. */ + * specifier part of it, without the prefix type byte. Return -1 if it is not + * a valid algorithm ID. */ static int secret_to_key_spec_len(uint8_t type) { @@ -86,7 +87,8 @@ secret_to_key_key_len(uint8_t type) case S2K_TYPE_SCRYPT: return DIGEST256_LEN; default: - return -1; + tor_fragile_assert(); // LCOV_EXCL_LINE + return -1; // LCOV_EXCL_LINE } } @@ -168,7 +170,7 @@ make_specifier(uint8_t *spec_out, uint8_t type, unsigned flags) spec_out[SCRYPT_SPEC_LEN-1] = (3u << 4) | (1u << 0); break; default: - tor_fragile_assert(); + tor_fragile_assert(); // LCOV_EXCL_LINE - we should have returned above. return S2K_BAD_ALGORITHM; } diff --git a/src/common/di_ops.c b/src/common/di_ops.c index 5dfe828066..4ed49e1164 100644 --- a/src/common/di_ops.c +++ b/src/common/di_ops.c @@ -226,3 +226,49 @@ safe_mem_is_zero(const void *mem, size_t sz) return 1 & ((total - 1) >> 8); } +/** Time-invariant 64-bit greater-than; works on two integers in the range + * (0,INT64_MAX). */ +#if SIZEOF_VOID_P == 8 +#define gt_i64_timei(a,b) ((a) > (b)) +#else +static inline int +gt_i64_timei(uint64_t a, uint64_t b) +{ + int64_t diff = (int64_t) (b - a); + int res = diff >> 63; + return res & 1; +} +#endif + +/** + * Given an array of list of <b>n_entries</b> uint64_t values, whose sum is + * <b>total</b>, find the first i such that the total of all elements 0...i is + * greater than rand_val. + * + * Try to perform this operation in a constant-time way. + */ +int +select_array_member_cumulative_timei(const uint64_t *entries, int n_entries, + uint64_t total, uint64_t rand_val) +{ + int i, i_chosen=-1, n_chosen=0; + uint64_t total_so_far = 0; + + for (i = 0; i < n_entries; ++i) { + total_so_far += entries[i]; + if (gt_i64_timei(total_so_far, rand_val)) { + i_chosen = i; + n_chosen++; + /* Set rand_val to INT64_MAX rather than stopping the loop. This way, + * the time we spend in the loop does not leak which element we chose. */ + rand_val = INT64_MAX; + } + } + tor_assert(total_so_far == total); + tor_assert(n_chosen == 1); + tor_assert(i_chosen >= 0); + tor_assert(i_chosen < n_entries); + + return i_chosen; +} + diff --git a/src/common/di_ops.h b/src/common/di_ops.h index 6e77b5cfd7..0a154302bf 100644 --- a/src/common/di_ops.h +++ b/src/common/di_ops.h @@ -42,6 +42,9 @@ void dimap_add_entry(di_digest256_map_t **map, const uint8_t *key, void *val); void *dimap_search(const di_digest256_map_t *map, const uint8_t *key, void *dflt_val); +int select_array_member_cumulative_timei(const uint64_t *entries, + int n_entries, + uint64_t total, uint64_t rand_val); #endif diff --git a/src/common/handles.h b/src/common/handles.h new file mode 100644 index 0000000000..1ee2322579 --- /dev/null +++ b/src/common/handles.h @@ -0,0 +1,153 @@ +/* Copyright (c) 2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file handles.h + * \brief Macros for C weak-handle implementation. + * + * A 'handle' is a pointer to an object that is allowed to go away while + * the handle stays alive. When you dereference the handle, you might get + * the object, or you might get "NULL". + * + * Use this pattern when an object has a single obvious lifespan, so you don't + * want to use reference counting, but when other objects might need to refer + * to the first object without caring about its lifetime. + * + * To enable a type to have handles, add a HANDLE_ENTRY() field in its + * definition, as in: + * + * struct walrus { + * HANDLE_ENTRY(wlr, walrus); + * // ... + * }; + * + * And invoke HANDLE_DECL(wlr, walrus, [static]) to declare the handle + * manipulation functions (typically in a header): + * + * // opaque handle to walrus. + * typedef struct wlr_handle_t wlr_handle_t; + * + * // make a new handle + * struct wlr_handle_t *wlr_handle_new(struct walrus *); + * + * // release a handle + * void wlr_handle_free(wlr_handle_t *); + * + * // return the pointed-to walrus, or NULL. + * struct walrus *wlr_handle_get(wlr_handle_t *). + * + * // call this function when you're about to free the walrus; + * // it invalidates all handles. (IF YOU DON'T, YOU WILL HAVE + * // DANGLING REFERENCES) + * void wlr_handles_clear(struct walrus *); + * + * Finally, use HANDLE_IMPL() to define the above functions in some + * appropriate C file: HANDLE_IMPL(wlr, walrus, [static]) + * + **/ + +#ifndef TOR_HANDLE_H +#define TOR_HANDLE_H + +#include "orconfig.h" +#include "tor_queue.h" +#include "util.h" + +#define HANDLE_ENTRY(name, structname) \ + struct name ## _handle_head_t *handle_head + +#define HANDLE_DECL(name, structname, linkage) \ + typedef struct name ## _handle_t name ## _handle_t; \ + linkage name ## _handle_t *name ## _handle_new(struct structname *object); \ + linkage void name ## _handle_free(name ## _handle_t *); \ + linkage struct structname *name ## _handle_get(name ## _handle_t *); \ + linkage void name ## _handles_clear(struct structname *object); + +/* + * Implementation notes: there are lots of possible implementations here. We + * could keep a linked list of handles, each with a backpointer to the object, + * and set all of their backpointers to NULL when the object is freed. Or we + * could have the clear function invalidate the object, but not actually let + * the object get freed until the all the handles went away. We could even + * have a hash-table mapping unique identifiers to objects, and have each + * handle be a copy of the unique identifier. (We'll want to build that last + * one eventually if we want cross-process handles.) + * + * But instead we're opting for a single independent 'head' that knows how + * many handles there are, and where the object is (or isn't). This makes + * all of our functions O(1), and most as fast as a single pointer access. + * + * The handles themselves are opaque structures holding a pointer to the head. + * We could instead have each foo_handle_t* be identical to foo_handle_head_t + * *, and save some allocations ... but doing so would make handle leaks + * harder to debug. As it stands, every handle leak is a memory leak, and + * existing memory debugging tools should help with those. We can revisit + * this decision if handles are too slow. + */ + +#define HANDLE_IMPL(name, structname, linkage) \ + /* The 'head' object for a handle-accessible type. This object */ \ + /* persists for as long as the object, or any handles, exist. */ \ + typedef struct name ## _handle_head_t { \ + struct structname *object; /* pointed-to object, or NULL */ \ + unsigned int references; /* number of existing handles */ \ + } name ## _handle_head_t; \ + \ + struct name ## _handle_t { \ + struct name ## _handle_head_t *head; /* reference to the 'head'. */ \ + }; \ + \ + linkage struct name ## _handle_t * \ + name ## _handle_new(struct structname *object) \ + { \ + tor_assert(object); \ + name ## _handle_head_t *head = object->handle_head; \ + if (PREDICT_UNLIKELY(head == NULL)) { \ + head = object->handle_head = tor_malloc_zero(sizeof(*head)); \ + head->object = object; \ + } \ + name ## _handle_t *new_ref = tor_malloc_zero(sizeof(*new_ref)); \ + new_ref->head = head; \ + ++head->references; \ + return new_ref; \ + } \ + \ + linkage void \ + name ## _handle_free(struct name ## _handle_t *ref) \ + { \ + if (! ref) return; \ + name ## _handle_head_t *head = ref->head; \ + tor_assert(head); \ + --head->references; \ + tor_free(ref); \ + if (head->object == NULL && head->references == 0) { \ + tor_free(head); \ + return; \ + } \ + } \ + \ + linkage struct structname * \ + name ## _handle_get(struct name ## _handle_t *ref) \ + { \ + tor_assert(ref); \ + name ## _handle_head_t *head = ref->head; \ + tor_assert(head); \ + return head->object; \ + } \ + \ + linkage void \ + name ## _handles_clear(struct structname *object) \ + { \ + tor_assert(object); \ + name ## _handle_head_t *head = object->handle_head; \ + if (! head) \ + return; \ + object->handle_head = NULL; \ + head->object = NULL; \ + if (head->references == 0) { \ + tor_free(head); \ + } \ + } + +#endif /* TOR_HANDLE_H */ + diff --git a/src/common/include.am b/src/common/include.am index 5afb30da6a..222afe0291 100644 --- a/src/common/include.am +++ b/src/common/include.am @@ -1,12 +1,14 @@ noinst_LIBRARIES += \ src/common/libor.a \ + src/common/libor-ctime.a \ src/common/libor-crypto.a \ src/common/libor-event.a if UNITTESTS_ENABLED noinst_LIBRARIES += \ src/common/libor-testing.a \ + src/common/libor-ctime-testing.a \ src/common/libor-crypto-testing.a \ src/common/libor-event-testing.a endif @@ -27,12 +29,14 @@ src_common_libcurve25519_donna_a_CFLAGS= if BUILD_CURVE25519_DONNA src_common_libcurve25519_donna_a_SOURCES=\ src/ext/curve25519_donna/curve25519-donna.c +# See bug 13538 -- this code is known to have signed overflow issues. src_common_libcurve25519_donna_a_CFLAGS+=\ - @F_OMIT_FRAME_POINTER@ + @F_OMIT_FRAME_POINTER@ @CFLAGS_CONSTTIME@ noinst_LIBRARIES+=src/common/libcurve25519_donna.a LIBDONNA=src/common/libcurve25519_donna.a else if BUILD_CURVE25519_DONNA_C64 +src_common_libcurve25519_donna_a_CFLAGS+=@CFLAGS_CONSTTIME@ src_common_libcurve25519_donna_a_SOURCES=\ src/ext/curve25519_donna/curve25519-donna-c64.c noinst_LIBRARIES+=src/common/libcurve25519_donna.a @@ -58,22 +62,37 @@ else readpassphrase_source= endif -LIBOR_A_SOURCES = \ +if ADD_MULODI4 +mulodi4_source=src/ext/mulodi/mulodi4.c +else +mulodi4_source= +endif + +LIBOR_CTIME_A_SRC = \ + $(mulodi4_source) \ + src/ext/csiphash.c \ + src/common/di_ops.c + +src_common_libor_ctime_a_SOURCES = $(LIBOR_CTIME_A_SRC) +src_common_libor_ctime_testing_a_SOURCES = $(LIBOR_CTIME_A_SRC) +src_common_libor_ctime_a_CFLAGS = @CFLAGS_CONSTTIME@ +src_common_libor_ctime_testing_a_CFLAGS = @CFLAGS_CONSTTIME@ $(TEST_CFLAGS) + +LIBOR_A_SRC = \ src/common/address.c \ src/common/backtrace.c \ src/common/compat.c \ src/common/compat_threads.c \ src/common/container.c \ - src/common/di_ops.c \ src/common/log.c \ src/common/memarea.c \ + src/common/pubsub.c \ src/common/util.c \ + src/common/util_bug.c \ src/common/util_format.c \ src/common/util_process.c \ src/common/sandbox.c \ src/common/workqueue.c \ - src/ext/csiphash.c \ - src/ext/trunnel/trunnel.c \ $(libor_extra_source) \ $(threads_impl_source) \ $(readpassphrase_source) @@ -81,7 +100,7 @@ LIBOR_A_SOURCES = \ src/common/src_common_libor_testing_a-log.$(OBJEXT) \ src/common/log.$(OBJEXT): micro-revision.i -LIBOR_CRYPTO_A_SOURCES = \ +LIBOR_CRYPTO_A_SRC = \ src/common/aes.c \ src/common/crypto.c \ src/common/crypto_pwbox.c \ @@ -89,21 +108,22 @@ LIBOR_CRYPTO_A_SOURCES = \ src/common/crypto_format.c \ src/common/torgzip.c \ src/common/tortls.c \ - src/trunnel/pwbox.c \ src/common/crypto_curve25519.c \ src/common/crypto_ed25519.c -LIBOR_EVENT_A_SOURCES = \ +LIBOR_EVENT_A_SRC = \ src/common/compat_libevent.c \ - src/common/procmon.c + src/common/procmon.c \ + src/common/timers.c \ + src/ext/timeouts/timeout.c -src_common_libor_a_SOURCES = $(LIBOR_A_SOURCES) -src_common_libor_crypto_a_SOURCES = $(LIBOR_CRYPTO_A_SOURCES) -src_common_libor_event_a_SOURCES = $(LIBOR_EVENT_A_SOURCES) +src_common_libor_a_SOURCES = $(LIBOR_A_SRC) +src_common_libor_crypto_a_SOURCES = $(LIBOR_CRYPTO_A_SRC) +src_common_libor_event_a_SOURCES = $(LIBOR_EVENT_A_SRC) -src_common_libor_testing_a_SOURCES = $(LIBOR_A_SOURCES) -src_common_libor_crypto_testing_a_SOURCES = $(LIBOR_CRYPTO_A_SOURCES) -src_common_libor_event_testing_a_SOURCES = $(LIBOR_EVENT_A_SOURCES) +src_common_libor_testing_a_SOURCES = $(LIBOR_A_SRC) +src_common_libor_crypto_testing_a_SOURCES = $(LIBOR_CRYPTO_A_SRC) +src_common_libor_event_testing_a_SOURCES = $(LIBOR_EVENT_A_SRC) src_common_libor_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_common_libor_crypto_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) @@ -129,16 +149,20 @@ COMMONHEADERS = \ src/common/crypto_pwbox.h \ src/common/crypto_s2k.h \ src/common/di_ops.h \ + src/common/handles.h \ src/common/memarea.h \ src/common/linux_syscalls.inc \ src/common/procmon.h \ + src/common/pubsub.h \ src/common/sandbox.h \ src/common/testsupport.h \ + src/common/timers.h \ src/common/torgzip.h \ src/common/torint.h \ src/common/torlog.h \ src/common/tortls.h \ src/common/util.h \ + src/common/util_bug.h \ src/common/util_format.h \ src/common/util_process.h \ src/common/workqueue.h diff --git a/src/common/log.c b/src/common/log.c index 6c387c6244..51309aa472 100644 --- a/src/common/log.c +++ b/src/common/log.c @@ -75,7 +75,7 @@ sev_to_string(int severity) case LOG_ERR: return "err"; default: /* Call assert, not tor_assert, since tor_assert * calls log on failure. */ - assert(0); return "UNKNOWN"; + assert(0); return "UNKNOWN"; // LCOV_EXCL_LINE } } @@ -95,7 +95,7 @@ should_log_function_name(log_domain_mask_t domain, int severity) return (domain & (LD_BUG|LD_NOFUNCNAME)) == LD_BUG; default: /* Call assert, not tor_assert, since tor_assert calls log on failure. */ - assert(0); return 0; + assert(0); return 0; // LCOV_EXCL_LINE } } @@ -270,7 +270,7 @@ log_tor_version(logfile_t *lf, int reset) return 0; } -const char bug_suffix[] = " (on Tor " VERSION +static const char bug_suffix[] = " (on Tor " VERSION #ifndef _MSC_VER " " #include "micro-revision.i" diff --git a/src/common/memarea.c b/src/common/memarea.c index 173ed4e1cb..7d16b702e3 100644 --- a/src/common/memarea.c +++ b/src/common/memarea.c @@ -131,7 +131,7 @@ alloc_chunk(size_t sz) /** Release <b>chunk</b> from a memarea. */ static void -chunk_free_unchecked(memarea_chunk_t *chunk) +memarea_chunk_free_unchecked(memarea_chunk_t *chunk) { CHECK_SENTINEL(chunk); tor_free(chunk); @@ -154,7 +154,7 @@ memarea_drop_all(memarea_t *area) memarea_chunk_t *chunk, *next; for (chunk = area->first; chunk; chunk = next) { next = chunk->next_chunk; - chunk_free_unchecked(chunk); + memarea_chunk_free_unchecked(chunk); } area->first = NULL; /*fail fast on */ tor_free(area); @@ -170,7 +170,7 @@ memarea_clear(memarea_t *area) if (area->first->next_chunk) { for (chunk = area->first->next_chunk; chunk; chunk = next) { next = chunk->next_chunk; - chunk_free_unchecked(chunk); + memarea_chunk_free_unchecked(chunk); } area->first->next_chunk = NULL; } diff --git a/src/common/procmon.c b/src/common/procmon.c index 12d53fcd41..4ecee26e8d 100644 --- a/src/common/procmon.c +++ b/src/common/procmon.c @@ -116,11 +116,11 @@ struct tor_process_monitor_t { * periodically check whether the process we have a handle to has * ended. */ HANDLE hproc; - /* XXX023 We can and should have Libevent watch hproc for us, - * if/when some version of Libevent 2.x can be told to do so. */ + /* XXXX We should have Libevent watch hproc for us, + * if/when some version of Libevent can be told to do so. */ #endif - /* XXX023 On Linux, we can and should receive the 22nd + /* XXXX On Linux, we can and should receive the 22nd * (space-delimited) field (‘starttime’) of /proc/$PID/stat from the * owning controller and store it, and poll once in a while to see * whether it has changed -- if so, the kernel has *definitely* @@ -130,7 +130,8 @@ struct tor_process_monitor_t { * systems whose admins have mounted procfs, or the start-time field * of the process-information structure returned by kvmgetprocs() on * any system. The latter is ickier. */ - /* XXX023 On FreeBSD (and possibly other kqueue systems), we can and + + /* XXXX On FreeBSD (and possibly other kqueue systems), we can and * should arrange to receive EVFILT_PROC NOTE_EXIT notifications for * pid, so we don't have to do such a heavyweight poll operation in * order to avoid the PID-reassignment race condition. (We would diff --git a/src/common/pubsub.c b/src/common/pubsub.c new file mode 100644 index 0000000000..b3faf40e00 --- /dev/null +++ b/src/common/pubsub.c @@ -0,0 +1,129 @@ +/* Copyright (c) 2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file pubsub.c + * + * \brief DOCDOC + */ + +#include "orconfig.h" +#include "pubsub.h" +#include "container.h" + +/** Helper: insert <b>s</b> into <b>topic's</b> list of subscribers, keeping + * them sorted in priority order. */ +static void +subscriber_insert(pubsub_topic_t *topic, pubsub_subscriber_t *s) +{ + int i; + smartlist_t *sl = topic->subscribers; + for (i = 0; i < smartlist_len(sl); ++i) { + pubsub_subscriber_t *other = smartlist_get(sl, i); + if (s->priority < other->priority) { + break; + } + } + smartlist_insert(sl, i, s); +} + +/** + * Add a new subscriber to <b>topic</b>, where (when an event is triggered), + * we'll notify the function <b>fn</b> by passing it <b>subscriber_data</b>. + * Return a handle to the subscribe which can later be passed to + * pubsub_unsubscribe_(). + * + * Functions are called in priority order, from lowest to highest. + * + * See pubsub.h for <b>subscribe_flags</b>. + */ +const pubsub_subscriber_t * +pubsub_subscribe_(pubsub_topic_t *topic, + pubsub_subscriber_fn_t fn, + void *subscriber_data, + unsigned subscribe_flags, + unsigned priority) +{ + tor_assert(! topic->locked); + if (subscribe_flags & SUBSCRIBE_ATSTART) { + tor_assert(topic->n_events_fired == 0); + } + pubsub_subscriber_t *r = tor_malloc_zero(sizeof(*r)); + r->priority = priority; + r->subscriber_flags = subscribe_flags; + r->fn = fn; + r->subscriber_data = subscriber_data; + if (topic->subscribers == NULL) { + topic->subscribers = smartlist_new(); + } + subscriber_insert(topic, r); + return r; +} + +/** + * Remove the subscriber <b>s</b> from <b>topic</b>. After calling this + * function, <b>s</b> may no longer be used. + */ +int +pubsub_unsubscribe_(pubsub_topic_t *topic, + const pubsub_subscriber_t *s) +{ + tor_assert(! topic->locked); + smartlist_t *sl = topic->subscribers; + if (sl == NULL) + return -1; + int i = smartlist_pos(sl, s); + if (i == -1) + return -1; + pubsub_subscriber_t *tmp = smartlist_get(sl, i); + tor_assert(tmp == s); + smartlist_del_keeporder(sl, i); + tor_free(tmp); + return 0; +} + +/** + * For every subscriber s in <b>topic</b>, invoke notify_fn on s and + * event_data. Return 0 if there were no nonzero return values, and -1 if + * there were any. + */ +int +pubsub_notify_(pubsub_topic_t *topic, pubsub_notify_fn_t notify_fn, + void *event_data, unsigned notify_flags) +{ + tor_assert(! topic->locked); + (void) notify_flags; + smartlist_t *sl = topic->subscribers; + int n_bad = 0; + ++topic->n_events_fired; + if (sl == NULL) + return -1; + topic->locked = 1; + SMARTLIST_FOREACH_BEGIN(sl, pubsub_subscriber_t *, s) { + int r = notify_fn(s, event_data); + if (r != 0) + ++n_bad; + } SMARTLIST_FOREACH_END(s); + topic->locked = 0; + return (n_bad == 0) ? 0 : -1; +} + +/** + * Release all storage held by <b>topic</b>. + */ +void +pubsub_clear_(pubsub_topic_t *topic) +{ + tor_assert(! topic->locked); + + smartlist_t *sl = topic->subscribers; + if (sl == NULL) + return; + SMARTLIST_FOREACH_BEGIN(sl, pubsub_subscriber_t *, s) { + tor_free(s); + } SMARTLIST_FOREACH_END(s); + smartlist_free(sl); + topic->subscribers = NULL; + topic->n_events_fired = 0; +} + diff --git a/src/common/pubsub.h b/src/common/pubsub.h new file mode 100644 index 0000000000..bbb4f02a42 --- /dev/null +++ b/src/common/pubsub.h @@ -0,0 +1,179 @@ +/* Copyright (c) 2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file pubsub.h + * \brief Macros to implement publish/subscribe abstractions. + * + * To use these macros, call DECLARE_PUBSUB_TOPIC() with an identifier to use + * as your topic. Below, I'm going to assume you say DECLARE_PUBSUB_TOPIC(T). + * + * Doing this will declare the following types: + * typedef struct T_event_data_t T_event_data_t; // you define this struct + * typedef struct T_subscriber_data_t T_subscriber_data_t; // this one too. + * typedef struct T_subscriber_t T_subscriber_t; // opaque + * typedef int (*T_subscriber_fn_t)(T_event_data_t*, T_subscriber_data_t*); + * + * and it will declare the following functions: + * const T_subscriber_t *T_subscribe(T_subscriber_fn_t, + * T_subscriber_data_t *, + * unsigned flags, + * unsigned priority); + * int T_unsubscribe(const T_subscriber_t *) + * + * Elsewhere you can say DECLARE_NOTIFY_PUBSUB_TOPIC(static, T), which + * declares: + * + * static int T_notify(T_event_data_t *, unsigned notify_flags); + * static void T_clear(void); + * + * And in some C file, you would define these functions with: + * IMPLEMENT_PUBSUB_TOPIC(static, T). + * + * The implementations will be small typesafe wrappers over generic versions + * of the above functions. + * + * To use the typesafe functions, you add any number of subscribers with + * T_subscribe(). Each has an associated function pointer, data pointer, + * and priority. Later, you can invoke T_notify() to declare that the + * event has occurred. Each of the subscribers will be invoked once. + **/ + +#ifndef TOR_PUBSUB_H +#define TOR_PUBSUB_H + +#include "torint.h" + +/** + * Flag for T_subscribe: die with an assertion failure if the event + * have ever been published before. Used when a subscriber must absolutely + * never have missed an event. + */ +#define SUBSCRIBE_ATSTART (1u<<0) + +#define DECLARE_PUBSUB_STRUCT_TYPES(name) \ + /* You define this type. */ \ + typedef struct name ## _event_data_t name ## _event_data_t; \ + /* You define this type. */ \ + typedef struct name ## _subscriber_data_t name ## _subscriber_data_t; + +#define DECLARE_PUBSUB_TOPIC(name) \ + /* This type is opaque. */ \ + typedef struct name ## _subscriber_t name ## _subscriber_t; \ + /* You declare functions matching this type. */ \ + typedef int (*name ## _subscriber_fn_t)( \ + name ## _event_data_t *data, \ + name ## _subscriber_data_t *extra); \ + /* Call this function to subscribe to a topic. */ \ + const name ## _subscriber_t *name ## _subscribe( \ + name##_subscriber_fn_t subscriber, \ + name##_subscriber_data_t *extra_data, \ + unsigned flags, \ + unsigned priority); \ + /* Call this function to unsubscribe from a topic. */ \ + int name ## _unsubscribe(const name##_subscriber_t *s); + +#define DECLARE_NOTIFY_PUBSUB_TOPIC(linkage, name) \ + /* Call this function to notify all subscribers. Flags not yet used. */ \ + linkage int name ## _notify(name ## _event_data_t *data, unsigned flags); \ + /* Call this function to release storage held by the topic. */ \ + linkage void name ## _clear(void); + +/** + * Type used to hold a generic function for a subscriber. + * + * [Yes, it is safe to cast to this, so long as we cast back to the original + * type before calling. From C99: "A pointer to a function of one type may be + * converted to a pointer to a function of another type and back again; the + * result shall compare equal to the original pointer."] +*/ +typedef int (*pubsub_subscriber_fn_t)(void *, void *); + +/** + * Helper type to implement pubsub abstraction. Don't use this directly. + * It represents a subscriber. + */ +typedef struct pubsub_subscriber_t { + /** Function to invoke when the event triggers. */ + pubsub_subscriber_fn_t fn; + /** Data associated with this subscriber. */ + void *subscriber_data; + /** Priority for this subscriber. Low priorities happen first. */ + unsigned priority; + /** Flags set on this subscriber. Not yet used.*/ + unsigned subscriber_flags; +} pubsub_subscriber_t; + +/** + * Helper type to implement pubsub abstraction. Don't use this directly. + * It represents a topic, and keeps a record of subscribers. + */ +typedef struct pubsub_topic_t { + /** List of subscribers to this topic. May be NULL. */ + struct smartlist_t *subscribers; + /** Total number of times that pubsub_notify_() has ever been called on this + * topic. */ + uint64_t n_events_fired; + /** True iff we're running 'notify' on this topic, and shouldn't allow + * any concurrent modifications or events. */ + unsigned locked; +} pubsub_topic_t; + +const pubsub_subscriber_t *pubsub_subscribe_(pubsub_topic_t *topic, + pubsub_subscriber_fn_t fn, + void *subscriber_data, + unsigned subscribe_flags, + unsigned priority); +int pubsub_unsubscribe_(pubsub_topic_t *topic, const pubsub_subscriber_t *sub); +void pubsub_clear_(pubsub_topic_t *topic); +typedef int (*pubsub_notify_fn_t)(pubsub_subscriber_t *subscriber, + void *notify_data); +int pubsub_notify_(pubsub_topic_t *topic, pubsub_notify_fn_t notify_fn, + void *notify_data, unsigned notify_flags); + +#define IMPLEMENT_PUBSUB_TOPIC(notify_linkage, name) \ + static pubsub_topic_t name ## _topic_ = { NULL, 0, 0 }; \ + const name ## _subscriber_t * \ + name ## _subscribe(name##_subscriber_fn_t subscriber, \ + name##_subscriber_data_t *extra_data, \ + unsigned flags, \ + unsigned priority) \ + { \ + const pubsub_subscriber_t *s; \ + s = pubsub_subscribe_(&name##_topic_, \ + (pubsub_subscriber_fn_t)subscriber, \ + extra_data, \ + flags, \ + priority); \ + return (const name##_subscriber_t *)s; \ + } \ + int \ + name ## _unsubscribe(const name##_subscriber_t *subscriber) \ + { \ + return pubsub_unsubscribe_(&name##_topic_, \ + (const pubsub_subscriber_t *)subscriber); \ + } \ + static int \ + name##_call_the_notify_fn_(pubsub_subscriber_t *subscriber, \ + void *notify_data) \ + { \ + name ## _subscriber_fn_t fn; \ + fn = (name ## _subscriber_fn_t) subscriber->fn; \ + return fn(notify_data, subscriber->subscriber_data); \ + } \ + notify_linkage int \ + name ## _notify(name ## _event_data_t *event_data, unsigned flags) \ + { \ + return pubsub_notify_(&name##_topic_, \ + name##_call_the_notify_fn_, \ + event_data, \ + flags); \ + } \ + notify_linkage void \ + name ## _clear(void) \ + { \ + pubsub_clear_(&name##_topic_); \ + } + +#endif /* TOR_PUBSUB_H */ + diff --git a/src/common/sandbox.c b/src/common/sandbox.c index 70c5bbd07c..838b267ab4 100644 --- a/src/common/sandbox.c +++ b/src/common/sandbox.c @@ -39,8 +39,6 @@ #if defined(USE_LIBSECCOMP) -#define _GNU_SOURCE - #include <sys/mman.h> #include <sys/syscall.h> #include <sys/types.h> @@ -1241,7 +1239,7 @@ prot_strings(scmp_filter_ctx ctx, sandbox_cfg_t* cfg) /** * Auxiliary function used in order to allocate a sandbox_cfg_t element and set - * it's values according the the parameter list. All elements are initialised + * its values according the parameter list. All elements are initialised * with the 'prot' field set to false, as the pointer is not protected at this * point. */ @@ -1443,7 +1441,7 @@ static HT_HEAD(getaddrinfo_cache, cached_getaddrinfo_item_t) HT_PROTOTYPE(getaddrinfo_cache, cached_getaddrinfo_item_t, node, cached_getaddrinfo_item_hash, - cached_getaddrinfo_items_eq); + cached_getaddrinfo_items_eq) HT_GENERATE2(getaddrinfo_cache, cached_getaddrinfo_item_t, node, cached_getaddrinfo_item_hash, cached_getaddrinfo_items_eq, diff --git a/src/common/sandbox.h b/src/common/sandbox.h index 2defd8bbd4..c5963e3119 100644 --- a/src/common/sandbox.h +++ b/src/common/sandbox.h @@ -39,12 +39,6 @@ typedef struct sandbox_cfg_elem sandbox_cfg_t; */ #ifdef USE_LIBSECCOMP -#ifndef __USE_GNU -#define __USE_GNU -#endif -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#endif #include <sys/ucontext.h> #include <seccomp.h> #include <netdb.h> diff --git a/src/common/testsupport.h b/src/common/testsupport.h index 3bb11a7e41..9ad2ba77e0 100644 --- a/src/common/testsupport.h +++ b/src/common/testsupport.h @@ -6,8 +6,10 @@ #ifdef TOR_UNIT_TESTS #define STATIC +#define EXTERN(type, name) extern type name; #else #define STATIC static +#define EXTERN(type, name) #endif /** Quick and dirty macros to implement test mocking. @@ -60,6 +62,12 @@ #define MOCK_IMPL(rv, funcname, arglist) \ rv(*funcname) arglist = funcname ##__real; \ rv funcname ##__real arglist +#define MOCK_DECL_ATTR(rv, funcname, arglist, attr) \ + rv funcname ##__real arglist attr; \ + extern rv(*funcname) arglist +#define MOCK_IMPL(rv, funcname, arglist) \ + rv(*funcname) arglist = funcname ##__real; \ + rv funcname ##__real arglist #define MOCK(func, replacement) \ do { \ (func) = (replacement); \ @@ -71,6 +79,8 @@ #else #define MOCK_DECL(rv, funcname, arglist) \ rv funcname arglist +#define MOCK_DECL_ATTR(rv, funcname, arglist, attr) \ + rv funcname arglist attr #define MOCK_IMPL(rv, funcname, arglist) \ rv funcname arglist #endif diff --git a/src/common/timers.c b/src/common/timers.c new file mode 100644 index 0000000000..5d8d1feafd --- /dev/null +++ b/src/common/timers.c @@ -0,0 +1,297 @@ +/* Copyright (c) 2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file timers.c + * \brief Wrapper around William Ahern's fast hierarchical timer wheel + * implementation, to tie it in with a libevent backend. + * + * Only use these functions from the main thread. + * + * The main advantage of tor_timer_t over using libevent's timers is that + * they're way more efficient if we need to have thousands or millions of + * them. For more information, see + * http://www.25thandclement.com/~william/projects/timeout.c.html + * + * Periodic timers are available in the backend, but I've turned them off. + * We can turn them back on if needed. + */ + +/* Notes: + * + * The use of tor_gettimeofday_cached_monotonic() is kind of ugly. It would + * be neat to fix it. + * + * Having a way to free all timers on shutdown would free people from the + * need to track them. Not sure if that's clever though. + * + * In an ideal world, Libevent would just switch to use this backend, and we + * could throw this file away. But even if Libevent does switch, we'll be + * stuck with legacy libevents for some time. + */ + +#include "orconfig.h" + +#include "compat.h" +#include "compat_libevent.h" +#include "timers.h" +#include "torlog.h" +#include "util.h" + +#ifdef HAVE_EVENT2_EVENT_H +#include <event2/event.h> +#else +#include <event.h> +#endif + +struct timeout_cb { + timer_cb_fn_t cb; + void *arg; +}; + +/* + * These definitions are for timeouts.c and timeouts.h. + */ +#ifdef __GNUC__ +/* We're not exposing any of the functions outside this file. */ +#define TIMEOUT_PUBLIC __attribute__((__unused__)) static +#else +/* We're not exposing any of the functions outside this file. */ +#define TIMEOUT_PUBLIC static +#endif +/* We're not using periodic events. */ +#define TIMEOUT_DISABLE_INTERVALS +/* We always know the global_timeouts object, so we don't need each timeout + * to keep a pointer to it. */ +#define TIMEOUT_DISABLE_RELATIVE_ACCESS +/* We're providing our own struct timeout_cb. */ +#define TIMEOUT_CB_OVERRIDE +/* We're going to support timers that are pretty far out in advance. Making + * this big can be inefficient, but having a significant number of timers + * above TIMEOUT_MAX can also be super-inefficent. Choosing 5 here sets + * timeout_max to 2^30 ticks, or 29 hours with our value for USEC_PER_TICK */ +#define WHEEL_NUM 5 +#include "src/ext/timeouts/timeout.c" + +static struct timeouts *global_timeouts = NULL; +static struct event *global_timer_event = NULL; + +/** We need to choose this value carefully. Because we're using timer wheels, + * it actually costs us to have extra resolution we don't use. So for now, + * I'm going to define our resolution as .1 msec, and hope that's good enough. + * + * Note that two of the most popular libevent backends (epoll without timerfd, + * and windows select), simply can't support sub-millisecond resolution, + * do this is optimistic for a lot of users. + */ +#define USEC_PER_TICK 100 + +/** One million microseconds in a second */ +#define USEC_PER_SEC 1000000 + +/** Check at least once every N seconds. */ +#define MIN_CHECK_SECONDS 3600 + +/** Check at least once every N ticks. */ +#define MIN_CHECK_TICKS \ + (((timeout_t)MIN_CHECK_SECONDS) * (1000000 / USEC_PER_TICK)) + +/** + * Convert the timeval in <b>tv</b> to a timeout_t, and return it. + * + * The output resolution is set by USEC_PER_TICK, and the time corresponding + * to 0 is the same as the time corresponding to 0 from + * tor_gettimeofday_cached_monotonic(). + */ +static timeout_t +tv_to_timeout(const struct timeval *tv) +{ + uint64_t usec = tv->tv_usec; + usec += ((uint64_t)USEC_PER_SEC) * tv->tv_sec; + return usec / USEC_PER_TICK; +} + +/** + * Convert the timeout in <b>t</b> to a timeval in <b>tv_out</b> + */ +static void +timeout_to_tv(timeout_t t, struct timeval *tv_out) +{ + t *= USEC_PER_TICK; + tv_out->tv_usec = (int)(t % USEC_PER_SEC); + tv_out->tv_sec = (time_t)(t / USEC_PER_SEC); +} + +/** + * Update the timer <b>tv</b> to the current time in <b>tv</b>. + */ +static void +timer_advance_to_cur_time(const struct timeval *tv) +{ + timeout_t cur_tick = tv_to_timeout(tv); + if (BUG(cur_tick < timeouts_get_curtime(global_timeouts))) { + cur_tick = timeouts_get_curtime(global_timeouts); // LCOV_EXCL_LINE + } + timeouts_update(global_timeouts, cur_tick); +} + +/** + * Adjust the time at which the libevent timer should fire based on + * the next-expiring time in <b>global_timeouts</b> + */ +static void +libevent_timer_reschedule(void) +{ + struct timeval now; + tor_gettimeofday_cached_monotonic(&now); + timer_advance_to_cur_time(&now); + + timeout_t delay = timeouts_timeout(global_timeouts); + struct timeval d; + if (delay > MIN_CHECK_TICKS) + delay = MIN_CHECK_TICKS; + timeout_to_tv(delay, &d); + event_add(global_timer_event, &d); +} + +/** + * Invoked when the libevent timer has expired: see which tor_timer_t events + * have fired, activate their callbacks, and reschedule the libevent timer. + */ +static void +libevent_timer_callback(evutil_socket_t fd, short what, void *arg) +{ + (void)fd; + (void)what; + (void)arg; + + struct timeval now; + tor_gettimeofday_cache_clear(); + tor_gettimeofday_cached_monotonic(&now); + timer_advance_to_cur_time(&now); + + tor_timer_t *t; + while ((t = timeouts_get(global_timeouts))) { + t->callback.cb(t, t->callback.arg, &now); + } + + tor_gettimeofday_cache_clear(); + libevent_timer_reschedule(); +} + +/** + * Initialize the timers subsystem. Requires that libevent has already been + * initialized. + */ +void +timers_initialize(void) +{ + if (BUG(global_timeouts)) + return; // LCOV_EXCL_LINE + + timeout_error_t err; + global_timeouts = timeouts_open(0, &err); + if (!global_timeouts) { + // LCOV_EXCL_START -- this can only fail on malloc failure. + log_err(LD_BUG, "Unable to open timer backend: %s", strerror(err)); + tor_assert(0); + // LCOV_EXCL_STOP + } + + struct event *timer_event; + timer_event = tor_event_new(tor_libevent_get_base(), + -1, 0, libevent_timer_callback, NULL); + tor_assert(timer_event); + global_timer_event = timer_event; + + libevent_timer_reschedule(); +} + +/** + * Release all storage held in the timers subsystem. Does not fire timers. + */ +void +timers_shutdown(void) +{ + if (global_timer_event) { + tor_event_free(global_timer_event); + global_timer_event = NULL; + } + if (global_timeouts) { + timeouts_close(global_timeouts); + global_timeouts = NULL; + } +} + +/** + * Allocate and return a new timer, with given callback and argument. + */ +tor_timer_t * +timer_new(timer_cb_fn_t cb, void *arg) +{ + tor_timer_t *t = tor_malloc(sizeof(tor_timer_t)); + timeout_init(t, 0); + timer_set_cb(t, cb, arg); + return t; +} + +/** + * Release all storage held by <b>t</b>, and unschedule it if was already + * scheduled. + */ +void +timer_free(tor_timer_t *t) +{ + if (! t) + return; + + timeouts_del(global_timeouts, t); + tor_free(t); +} + +/** + * Change the callback and argument associated with a timer <b>t</b>. + */ +void +timer_set_cb(tor_timer_t *t, timer_cb_fn_t cb, void *arg) +{ + t->callback.cb = cb; + t->callback.arg = arg; +} + +/** + * Schedule the timer t to fire at the current time plus a delay of <b>tv</b>. + * All times are relative to tor_gettimeofday_cached_monotonic. + */ +void +timer_schedule(tor_timer_t *t, const struct timeval *tv) +{ + const timeout_t when = tv_to_timeout(tv); + struct timeval now; + tor_gettimeofday_cached_monotonic(&now); + timer_advance_to_cur_time(&now); + + /* Take the old timeout value. */ + timeout_t to = timeouts_timeout(global_timeouts); + + timeouts_add(global_timeouts, t, when); + + /* Should we update the libevent timer? */ + if (to <= when) { + return; /* we're already going to fire before this timer would trigger. */ + } + libevent_timer_reschedule(); +} + +/** + * Cancel the timer <b>t</b> if it is currently scheduled. (It's okay to call + * this on an unscheduled timer. + */ +void +timer_disable(tor_timer_t *t) +{ + timeouts_del(global_timeouts, t); + /* We don't reschedule the libevent timer here, since it's okay if it fires + * early. */ +} + diff --git a/src/common/timers.h b/src/common/timers.h new file mode 100644 index 0000000000..594cf38a64 --- /dev/null +++ b/src/common/timers.h @@ -0,0 +1,22 @@ +/* Copyright (c) 2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_TIMERS_H +#define TOR_TIMERS_H + +#include "orconfig.h" +#include "testsupport.h" + +typedef struct timeout tor_timer_t; +typedef void (*timer_cb_fn_t)(tor_timer_t *, void *, const struct timeval *); +tor_timer_t *timer_new(timer_cb_fn_t cb, void *arg); +void timer_set_cb(tor_timer_t *t, timer_cb_fn_t cb, void *arg); +void timer_schedule(tor_timer_t *t, const struct timeval *delay); +void timer_disable(tor_timer_t *t); +void timer_free(tor_timer_t *t); + +void timers_initialize(void); +void timers_shutdown(void); + +#endif + diff --git a/src/common/torgzip.c b/src/common/torgzip.c index 71e55f8723..331bb5a017 100644 --- a/src/common/torgzip.c +++ b/src/common/torgzip.c @@ -46,34 +46,16 @@ #include <zlib.h> +#if defined ZLIB_VERNUM && ZLIB_VERNUM < 0x1200 +#error "We require zlib version 1.2 or later." +#endif + static size_t tor_zlib_state_size_precalc(int inflate, int windowbits, int memlevel); /** Total number of bytes allocated for zlib state */ static size_t total_zlib_allocation = 0; -/** Set to 1 if zlib is a version that supports gzip; set to 0 if it doesn't; - * set to -1 if we haven't checked yet. */ -static int gzip_is_supported = -1; - -/** Return true iff we support gzip-based compression. Otherwise, we need to - * use zlib. */ -int -is_gzip_supported(void) -{ - if (gzip_is_supported >= 0) - return gzip_is_supported; - - if (!strcmpstart(ZLIB_VERSION, "0.") || - !strcmpstart(ZLIB_VERSION, "1.0") || - !strcmpstart(ZLIB_VERSION, "1.1")) - gzip_is_supported = 0; - else - gzip_is_supported = 1; - - return gzip_is_supported; -} - /** Return a string representation of the version of the currently running * version of zlib. */ const char * @@ -165,12 +147,6 @@ tor_gzip_compress(char **out, size_t *out_len, *out = NULL; - if (method == GZIP_METHOD && !is_gzip_supported()) { - /* Old zlib version don't support gzip in deflateInit2 */ - log_warn(LD_BUG, "Gzip not supported with zlib %s", ZLIB_VERSION); - goto err; - } - stream = tor_malloc_zero(sizeof(struct z_stream_s)); stream->zalloc = Z_NULL; stream->zfree = Z_NULL; @@ -182,9 +158,11 @@ tor_gzip_compress(char **out, size_t *out_len, method_bits(method, HIGH_COMPRESSION), get_memlevel(HIGH_COMPRESSION), Z_DEFAULT_STRATEGY) != Z_OK) { + //LCOV_EXCL_START -- we can only provoke failure by giving junk arguments. log_warn(LD_GENERAL, "Error from deflateInit2: %s", stream->msg?stream->msg:"<no message>"); goto err; + //LCOV_EXCL_STOP } /* Guess 50% compression. */ @@ -237,13 +215,12 @@ tor_gzip_compress(char **out, size_t *out_len, * the newly unsigned field isn't negative." */ tor_assert(stream->total_out >= 0); #endif - if (((size_t)stream->total_out) > out_size + 4097) { - /* If we're wasting more than 4k, don't. */ - *out = tor_realloc(*out, stream->total_out + 1); - } if (deflateEnd(stream)!=Z_OK) { + // LCOV_EXCL_START -- unreachable if we handled the zlib structure right + tor_assert_nonfatal_unreached(); log_warn(LD_BUG, "Error freeing gzip structures"); goto err; + // LCOV_EXCL_STOP } tor_free(stream); @@ -291,12 +268,6 @@ tor_gzip_uncompress(char **out, size_t *out_len, tor_assert(in); tor_assert(in_len < UINT_MAX); - if (method == GZIP_METHOD && !is_gzip_supported()) { - /* Old zlib version don't support gzip in inflateInit2 */ - log_warn(LD_BUG, "Gzip not supported with zlib %s", ZLIB_VERSION); - return -1; - } - *out = NULL; stream = tor_malloc_zero(sizeof(struct z_stream_s)); @@ -308,9 +279,11 @@ tor_gzip_uncompress(char **out, size_t *out_len, if (inflateInit2(stream, method_bits(method, HIGH_COMPRESSION)) != Z_OK) { + // LCOV_EXCL_START -- can only hit this if we give bad inputs. log_warn(LD_GENERAL, "Error from inflateInit2: %s", stream->msg?stream->msg:"<no message>"); goto err; + // LCOV_EXCL_STOP } out_size = in_len * 2; /* guess 50% compression. */ @@ -451,12 +424,6 @@ tor_zlib_new(int compress, compress_method_t method, tor_zlib_state_t *out; int bits, memlevel; - if (method == GZIP_METHOD && !is_gzip_supported()) { - /* Old zlib version don't support gzip in inflateInit2 */ - log_warn(LD_BUG, "Gzip not supported with zlib %s", ZLIB_VERSION); - return NULL; - } - if (! compress) { /* use this setting for decompression, since we might have the * max number of window bits */ @@ -474,10 +441,10 @@ tor_zlib_new(int compress, compress_method_t method, if (deflateInit2(&out->stream, Z_BEST_COMPRESSION, Z_DEFLATED, bits, memlevel, Z_DEFAULT_STRATEGY) != Z_OK) - goto err; + goto err; // LCOV_EXCL_LINE } else { if (inflateInit2(&out->stream, bits) != Z_OK) - goto err; + goto err; // LCOV_EXCL_LINE } out->allocation = tor_zlib_state_size_precalc(!compress, bits, memlevel); diff --git a/src/common/torlog.h b/src/common/torlog.h index 578af7caea..80f37e0e48 100644 --- a/src/common/torlog.h +++ b/src/common/torlog.h @@ -176,7 +176,7 @@ void log_fn_ratelim_(struct ratelim_t *ratelim, int severity, const char *format, ...) CHECK_PRINTF(5,6); -#if defined(__GNUC__) +#if defined(__GNUC__) && __GNUC__ <= 3 /* These are the GCC varidaic macros, so that older versions of GCC don't * break. */ diff --git a/src/common/tortls.c b/src/common/tortls.c index b68f5dfcdf..0395205228 100644 --- a/src/common/tortls.c +++ b/src/common/tortls.c @@ -24,18 +24,11 @@ #include <ws2tcpip.h> #endif -#ifdef __GNUC__ -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -#endif +#include "compat.h" -#if __GNUC__ && GCC_VERSION >= 402 -#if GCC_VERSION >= 406 -#pragma GCC diagnostic push -#endif /* Some versions of OpenSSL declare SSL_get_selected_srtp_profile twice in * srtp.h. Suppress the GCC warning so we can build with -Wredundant-decl. */ -#pragma GCC diagnostic ignored "-Wredundant-decls" -#endif +DISABLE_GCC_WARNING(redundant-decls) #include <openssl/opensslv.h> #include "crypto.h" @@ -53,13 +46,7 @@ #include <openssl/bn.h> #include <openssl/rsa.h> -#if __GNUC__ && GCC_VERSION >= 402 -#if GCC_VERSION >= 406 -#pragma GCC diagnostic pop -#else -#pragma GCC diagnostic warning "-Wredundant-decls" -#endif -#endif +ENABLE_GCC_WARNING(redundant-decls) #ifdef USE_BUFFEREVENTS #include <event2/bufferevent_ssl.h> @@ -577,7 +564,7 @@ MOCK_IMPL(STATIC X509 *, /** List of ciphers that servers should select from when we actually have * our choice of what cipher to use. */ -const char UNRESTRICTED_SERVER_CIPHER_LIST[] = +static const char UNRESTRICTED_SERVER_CIPHER_LIST[] = /* This list is autogenerated with the gen_server_ciphers.py script; * don't hand-edit it. */ #ifdef TLS1_TXT_ECDHE_RSA_WITH_AES_256_GCM_SHA384 diff --git a/src/common/tortls.h b/src/common/tortls.h index 1a59c67df3..b6ab2ec8f5 100644 --- a/src/common/tortls.h +++ b/src/common/tortls.h @@ -164,8 +164,18 @@ STATIC int tor_tls_context_init_one(tor_tls_context_t **ppcontext, int is_client); STATIC void tls_log_errors(tor_tls_t *tls, int severity, int domain, const char *doing); + +#ifdef TOR_UNIT_TESTS +extern int tor_tls_object_ex_data_index; +extern tor_tls_context_t *server_tls_context; +extern tor_tls_context_t *client_tls_context; +extern uint16_t v2_cipher_list[]; +extern uint64_t total_bytes_written_over_tls; +extern uint64_t total_bytes_written_by_tls; #endif +#endif /* endif TORTLS_PRIVATE */ + const char *tor_tls_err_to_string(int err); void tor_tls_get_state_description(tor_tls_t *tls, char *buf, size_t sz); diff --git a/src/common/util.c b/src/common/util.c index f3effe0957..538aeb108d 100644 --- a/src/common/util.c +++ b/src/common/util.c @@ -9,10 +9,6 @@ * process control. **/ -/* This is required on rh7 to make strptime not complain. - */ -#define _GNU_SOURCE - #include "orconfig.h" #ifdef HAVE_FCNTL_H #include <fcntl.h> @@ -105,23 +101,6 @@ #endif /* ===== - * Assertion helper. - * ===== */ -/** Helper for tor_assert: report the assertion failure. */ -void -tor_assertion_failed_(const char *fname, unsigned int line, - const char *func, const char *expr) -{ - char buf[256]; - log_err(LD_BUG, "%s:%u: %s: Assertion %s failed; aborting.", - fname, line, func, expr); - tor_snprintf(buf, sizeof(buf), - "Assertion %s failed in %s at %s:%u", - expr, func, fname, line); - log_backtrace(LOG_ERR, LD_BUG, buf); -} - -/* ===== * Memory management * ===== */ #ifdef USE_DMALLOC @@ -172,11 +151,13 @@ tor_malloc_(size_t size DMALLOC_PARAMS) #endif if (PREDICT_UNLIKELY(result == NULL)) { + /* LCOV_EXCL_START */ log_err(LD_MM,"Out of memory on malloc(). Dying."); /* If these functions die within a worker process, they won't call * spawn_exit, but that's ok, since the parent will run out of memory soon * anyway. */ exit(1); + /* LCOV_EXCL_STOP */ } return result; } @@ -221,6 +202,15 @@ size_mul_check(const size_t x, const size_t y) x <= SIZE_MAX / y); } +#ifdef TOR_UNIT_TESTS +/** Exposed for unit tests only */ +int +size_mul_check__(const size_t x, const size_t y) +{ + return size_mul_check(x,y); +} +#endif + /** Allocate a chunk of <b>nmemb</b>*<b>size</b> bytes of memory, fill * the memory with zero bytes, and return a pointer to the result. * Log and terminate the process on error. (Same as @@ -260,8 +250,10 @@ tor_realloc_(void *ptr, size_t size DMALLOC_PARAMS) #endif if (PREDICT_UNLIKELY(result == NULL)) { + /* LCOV_EXCL_START */ log_err(LD_MM,"Out of memory on realloc(). Dying."); exit(1); + /* LCOV_EXCL_STOP */ } return result; } @@ -296,8 +288,10 @@ tor_strdup_(const char *s DMALLOC_PARAMS) dup = strdup(s); #endif if (PREDICT_UNLIKELY(dup == NULL)) { + /* LCOV_EXCL_START */ log_err(LD_MM,"Out of memory on strdup(). Dying."); exit(1); + /* LCOV_EXCL_STOP */ } return dup; } @@ -359,6 +353,7 @@ tor_free_(void *mem) tor_free(mem); } +DISABLE_GCC_WARNING(aggregate-return) /** Call the platform malloc info function, and dump the results to the log at * level <b>severity</b>. If no such function exists, do nothing. */ void @@ -386,6 +381,7 @@ tor_log_mallinfo(int severity) ); #endif } +ENABLE_GCC_WARNING(aggregate-return) /* ===== * Math @@ -530,21 +526,6 @@ round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor) return number; } -/** Return the lowest x in [INT64_MIN, INT64_MAX] such that x is at least - * <b>number</b>, and x modulo <b>divisor</b> == 0. If no such x can be - * expressed as an int64_t, return INT64_MAX */ -int64_t -round_int64_to_next_multiple_of(int64_t number, int64_t divisor) -{ - tor_assert(divisor > 0); - if (INT64_MAX - divisor + 1 < number) - return INT64_MAX; - if (number >= 0) - number += divisor - 1; - number -= number % divisor; - return number; -} - /** Transform a random value <b>p</b> from the uniform distribution in * [0.0, 1.0[ into a Laplace distributed value with location parameter * <b>mu</b> and scale parameter <b>b</b>. Truncate the final result @@ -1130,6 +1111,9 @@ tor_digest256_is_zero(const char *digest) /* Were there unexpected unconverted characters? */ \ if (!next && *endptr) \ goto err; \ + /* Illogical (max, min) inputs? */ \ + if (BUG(max < min)) \ + goto err; \ /* Is r within limits? */ \ if (r < min || r > max) \ goto err; \ @@ -1402,42 +1386,138 @@ tor_escape_str_for_pt_args(const char *string, const char *chars_to_escape) * Time * ===== */ +#define TOR_USEC_PER_SEC 1000000 + +/** Return the difference between start->tv_sec and end->tv_sec. + * Returns INT64_MAX on overflow and underflow. + */ +static int64_t +tv_secdiff_impl(const struct timeval *start, const struct timeval *end) +{ + const int64_t s = (int64_t)start->tv_sec; + const int64_t e = (int64_t)end->tv_sec; + + /* This may not be the most efficient way of implemeting this check, + * but it's easy to see that it's correct and doesn't overflow */ + + if (s > 0 && e < INT64_MIN + s) { + /* s is positive: equivalent to e - s < INT64_MIN, but without any + * overflow */ + return INT64_MAX; + } else if (s < 0 && e > INT64_MAX + s) { + /* s is negative: equivalent to e - s > INT64_MAX, but without any + * overflow */ + return INT64_MAX; + } + + return e - s; +} + /** Return the number of microseconds elapsed between *start and *end. + * Returns LONG_MAX on overflow and underflow. */ long tv_udiff(const struct timeval *start, const struct timeval *end) { - long udiff; - long secdiff = end->tv_sec - start->tv_sec; + /* Sanity check tv_usec */ + if (start->tv_usec > TOR_USEC_PER_SEC || start->tv_usec < 0) { + log_warn(LD_GENERAL, "comparing times on microsecond detail with bad " + "start tv_usec: " I64_FORMAT " microseconds", + I64_PRINTF_ARG(start->tv_usec)); + return LONG_MAX; + } + + if (end->tv_usec > TOR_USEC_PER_SEC || end->tv_usec < 0) { + log_warn(LD_GENERAL, "comparing times on microsecond detail with bad " + "end tv_usec: " I64_FORMAT " microseconds", + I64_PRINTF_ARG(end->tv_usec)); + return LONG_MAX; + } + + /* Some BSDs have struct timeval.tv_sec 64-bit, but time_t (and long) 32-bit + */ + int64_t udiff; + const int64_t secdiff = tv_secdiff_impl(start, end); - if (labs(secdiff+1) > LONG_MAX/1000000) { + /* end->tv_usec - start->tv_usec can be up to 1 second either way */ + if (secdiff > (int64_t)(LONG_MAX/1000000 - 1) || + secdiff < (int64_t)(LONG_MIN/1000000 + 1)) { log_warn(LD_GENERAL, "comparing times on microsecond detail too far " - "apart: %ld seconds", secdiff); + "apart: " I64_FORMAT " seconds", I64_PRINTF_ARG(secdiff)); + return LONG_MAX; + } + + /* we'll never get an overflow here, because we check that both usecs are + * between 0 and TV_USEC_PER_SEC. */ + udiff = secdiff*1000000 + ((int64_t)end->tv_usec - (int64_t)start->tv_usec); + + /* Some compilers are smart enough to work out this is a no-op on L64 */ +#if SIZEOF_LONG < 8 + if (udiff > (int64_t)LONG_MAX || udiff < (int64_t)LONG_MIN) { return LONG_MAX; } +#endif - udiff = secdiff*1000000L + (end->tv_usec - start->tv_usec); - return udiff; + return (long)udiff; } /** Return the number of milliseconds elapsed between *start and *end. + * If the tv_usec difference is 500, rounds away from zero. + * Returns LONG_MAX on overflow and underflow. */ long tv_mdiff(const struct timeval *start, const struct timeval *end) { - long mdiff; - long secdiff = end->tv_sec - start->tv_sec; + /* Sanity check tv_usec */ + if (start->tv_usec > TOR_USEC_PER_SEC || start->tv_usec < 0) { + log_warn(LD_GENERAL, "comparing times on millisecond detail with bad " + "start tv_usec: " I64_FORMAT " microseconds", + I64_PRINTF_ARG(start->tv_usec)); + return LONG_MAX; + } + + if (end->tv_usec > TOR_USEC_PER_SEC || end->tv_usec < 0) { + log_warn(LD_GENERAL, "comparing times on millisecond detail with bad " + "end tv_usec: " I64_FORMAT " microseconds", + I64_PRINTF_ARG(end->tv_usec)); + return LONG_MAX; + } - if (labs(secdiff+1) > LONG_MAX/1000) { + /* Some BSDs have struct timeval.tv_sec 64-bit, but time_t (and long) 32-bit + */ + int64_t mdiff; + const int64_t secdiff = tv_secdiff_impl(start, end); + + /* end->tv_usec - start->tv_usec can be up to 1 second either way, but the + * mdiff calculation may add another temporary second for rounding. + * Whether this actually causes overflow depends on the compiler's constant + * folding and order of operations. */ + if (secdiff > (int64_t)(LONG_MAX/1000 - 2) || + secdiff < (int64_t)(LONG_MIN/1000 + 1)) { log_warn(LD_GENERAL, "comparing times on millisecond detail too far " - "apart: %ld seconds", secdiff); + "apart: " I64_FORMAT " seconds", I64_PRINTF_ARG(secdiff)); return LONG_MAX; } /* Subtract and round */ - mdiff = secdiff*1000L + - ((long)end->tv_usec - (long)start->tv_usec + 500L) / 1000L; - return mdiff; + mdiff = secdiff*1000 + + /* We add a million usec here to ensure that the result is positive, + * so that the round-towards-zero behavior of the division will give + * the right result for rounding to the nearest msec. Later we subtract + * 1000 in order to get the correct result. + * We'll never get an overflow here, because we check that both usecs are + * between 0 and TV_USEC_PER_SEC. */ + ((int64_t)end->tv_usec - (int64_t)start->tv_usec + 500 + 1000000) / 1000 + - 1000; + + /* Some compilers are smart enough to work out this is a no-op on L64 */ +#if SIZEOF_LONG < 8 + if (mdiff > (int64_t)LONG_MAX || mdiff < (int64_t)LONG_MIN) { + return LONG_MAX; + } +#endif + + return (long)mdiff; } /** @@ -1638,11 +1718,16 @@ parse_rfc1123_time(const char *buf, time_t *t) tm.tm_sec = (int)tm_sec; if (tm.tm_year < 1970) { + /* LCOV_EXCL_START + * XXXX I think this is dead code; we already checked for + * invalid_year above. */ + tor_assert_nonfatal_unreached(); char *esc = esc_for_log(buf); log_warn(LD_GENERAL, "Got invalid RFC1123 time %s. (Before 1970)", esc); tor_free(esc); return -1; + /* LCOV_EXCL_STOP */ } tm.tm_year -= 1900; @@ -1726,10 +1811,15 @@ parse_iso_time_(const char *cp, time_t *t, int strict) st_tm.tm_wday = 0; /* Should be ignored. */ if (st_tm.tm_year < 70) { + /* LCOV_EXCL_START + * XXXX I think this is dead code; we already checked for + * year < 1970 above. */ + tor_assert_nonfatal_unreached(); char *esc = esc_for_log(cp); log_warn(LD_GENERAL, "Got invalid ISO time %s. (Before 1970)", esc); tor_free(esc); return -1; + /* LCOV_EXCL_STOP */ } return tor_timegm(&st_tm, t); } @@ -1920,6 +2010,8 @@ rate_limit_log(ratelim_t *lim, time_t now) return tor_strdup(""); } else { char *cp=NULL; + /* XXXX this is not exactly correct: the messages could have occurred + * any time between the old value of lim->allowed and now. */ tor_asprintf(&cp, " [%d similar message(s) suppressed in last %d seconds]", n-1, lim->rate); @@ -2011,6 +2103,16 @@ clean_name_for_stat(char *name) #endif } +/** Wrapper for unlink() to make it mockable for the test suite; returns 0 + * if unlinking the file succeeded, -1 and sets errno if unlinking fails. + */ + +MOCK_IMPL(int, +tor_unlink,(const char *pathname)) +{ + return unlink(pathname); +} + /** Return: * FN_ERROR if filename can't be read, is NULL, or is zero-length, * FN_NOENT if it doesn't exist, @@ -2074,9 +2176,9 @@ file_status(const char *fname) * When effective_user is not NULL, check permissions against the given user * and its primary group. */ -int -check_private_dir(const char *dirname, cpd_check_t check, - const char *effective_user) +MOCK_IMPL(int, +check_private_dir,(const char *dirname, cpd_check_t check, + const char *effective_user)) { int r; struct stat st; @@ -2304,8 +2406,8 @@ check_private_dir(const char *dirname, cpd_check_t check, * function, and all other functions in util.c that create files, create them * with mode 0600. */ -int -write_str_to_file(const char *fname, const char *str, int bin) +MOCK_IMPL(int, +write_str_to_file,(const char *fname, const char *str, int bin)) { #ifdef _WIN32 if (!bin && strchr(str, '\r')) { @@ -2664,8 +2766,8 @@ read_file_to_str_until_eof(int fd, size_t max_bytes_to_read, size_t *sz_out) * the call to stat and the call to read_all: the resulting string will * be truncated. */ -char * -read_file_to_str(const char *filename, int flags, struct stat *stat_out) +MOCK_IMPL(char *, +read_file_to_str, (const char *filename, int flags, struct stat *stat_out)) { int fd; /* router file */ struct stat statbuf; @@ -2820,9 +2922,11 @@ unescape_string(const char *s, char **result, size_t *size_out) if (size_out) *size_out = out - *result; return cp+1; case '\0': + /* LCOV_EXCL_START -- we caught this in parse_config_from_line. */ tor_fragile_assert(); tor_free(*result); return NULL; + /* LCOV_EXCL_STOP */ case '\\': switch (cp[1]) { @@ -2836,8 +2940,12 @@ unescape_string(const char *s, char **result, size_t *size_out) x1 = hex_decode_digit(cp[2]); x2 = hex_decode_digit(cp[3]); if (x1 == -1 || x2 == -1) { - tor_free(*result); - return NULL; + /* LCOV_EXCL_START */ + /* we caught this above in the initial loop. */ + tor_assert_nonfatal_unreached(); + tor_free(*result); + return NULL; + /* LCOV_EXCL_STOP */ } *out++ = ((x1<<4) + x2); @@ -2863,7 +2971,11 @@ unescape_string(const char *s, char **result, size_t *size_out) cp += 2; break; default: + /* LCOV_EXCL_START */ + /* we caught this above in the initial loop. */ + tor_assert_nonfatal_unreached(); tor_free(*result); return NULL; + /* LCOV_EXCL_STOP */ } break; default: @@ -3079,7 +3191,7 @@ digit_to_num(char d) * success, store the result in <b>out</b>, advance bufp to the next * character, and return 0. On failure, return -1. */ static int -scan_unsigned(const char **bufp, unsigned long *out, int width, int base) +scan_unsigned(const char **bufp, unsigned long *out, int width, unsigned base) { unsigned long result = 0; int scanned_so_far = 0; @@ -3092,7 +3204,7 @@ scan_unsigned(const char **bufp, unsigned long *out, int width, int base) while (**bufp && (hex?TOR_ISXDIGIT(**bufp):TOR_ISDIGIT(**bufp)) && scanned_so_far < width) { - int digit = hex?hex_decode_digit(*(*bufp)++):digit_to_num(*(*bufp)++); + unsigned digit = hex?hex_decode_digit(*(*bufp)++):digit_to_num(*(*bufp)++); // Check for overflow beforehand, without actually causing any overflow // This preserves functionality on compilers that don't wrap overflow // (i.e. that trap or optimise away overflow) @@ -3138,14 +3250,15 @@ scan_signed(const char **bufp, long *out, int width) if (neg && result > 0) { if (result > ((unsigned long)LONG_MAX) + 1) return -1; /* Underflow */ - // Avoid overflow on the cast to signed long when result is LONG_MIN - // by subtracting 1 from the unsigned long positive value, - // then, after it has been cast to signed and negated, - // subtracting the original 1 (the double-subtraction is intentional). - // Otherwise, the cast to signed could cause a temporary long - // to equal LONG_MAX + 1, which is undefined. - // We avoid underflow on the subtraction by treating -0 as positive. - *out = (-(long)(result - 1)) - 1; + else if (result == ((unsigned long)LONG_MAX) + 1) + *out = LONG_MIN; + else { + /* We once had a far more clever no-overflow conversion here, but + * some versions of GCC apparently ran it into the ground. Now + * we just check for LONG_MIN explicitly. + */ + *out = -(long)result; + } } else { if (result > LONG_MAX) return -1; /* Overflow */ @@ -3291,8 +3404,10 @@ tor_vsscanf(const char *buf, const char *pattern, va_list ap) *out = lng; } else { int *out = va_arg(ap, int *); +#if LONG_MAX > INT_MAX if (lng < INT_MIN || lng > INT_MAX) return n_matched; +#endif *out = (int)lng; } ++pattern; @@ -3387,8 +3502,8 @@ smartlist_add_vasprintf(struct smartlist_t *sl, const char *pattern, /** Return a new list containing the filenames in the directory <b>dirname</b>. * Return NULL on error or if <b>dirname</b> is not a directory. */ -smartlist_t * -tor_listdir(const char *dirname) +MOCK_IMPL(smartlist_t *, +tor_listdir, (const char *dirname)) { smartlist_t *result; #ifdef _WIN32 @@ -3495,13 +3610,17 @@ start_daemon(void) start_daemon_called = 1; if (pipe(daemon_filedes)) { + /* LCOV_EXCL_START */ log_err(LD_GENERAL,"pipe failed; exiting. Error was %s", strerror(errno)); exit(1); + /* LCOV_EXCL_STOP */ } pid = fork(); if (pid < 0) { + /* LCOV_EXCL_START */ log_err(LD_GENERAL,"fork failed. Exiting."); exit(1); + /* LCOV_EXCL_STOP */ } if (pid) { /* Parent */ int ok; @@ -3563,8 +3682,10 @@ finish_daemon(const char *desired_cwd) nullfd = tor_open_cloexec("/dev/null", O_RDWR, 0); if (nullfd < 0) { + /* LCOV_EXCL_START */ log_err(LD_GENERAL,"/dev/null can't be opened. Exiting."); exit(1); + /* LCOV_EXCL_STOP */ } /* close fds linking to invoking terminal, but * close usual incoming fds, but redirect them somewhere @@ -3573,8 +3694,10 @@ finish_daemon(const char *desired_cwd) if (dup2(nullfd,0) < 0 || dup2(nullfd,1) < 0 || dup2(nullfd,2) < 0) { + /* LCOV_EXCL_START */ log_err(LD_GENERAL,"dup2 failed. Exiting."); exit(1); + /* LCOV_EXCL_STOP */ } if (nullfd > 2) close(nullfd); @@ -3771,7 +3894,7 @@ format_number_sigsafe(unsigned long x, char *buf, int buf_len, /* NOT tor_assert; see above. */ if (cp != buf) { - abort(); + abort(); // LCOV_EXCL_LINE } return len; @@ -4349,7 +4472,7 @@ tor_spawn_background(const char *const filename, const char **argv, _exit(255); /* Never reached, but avoids compiler warning */ - return status; + return status; // LCOV_EXCL_LINE } /* In parent */ @@ -5557,10 +5680,31 @@ clamp_double_to_int64(double number) * representable integer for which this is not the case is INT64_MIN, but * it is covered by the logic below. */ if (isfinite(number) && exp <= 63) { - return number; + return (int64_t)number; } /* Handle infinities and finite numbers with magnitude >= 2^63. */ return signbit(number) ? INT64_MIN : INT64_MAX; } +/** Return a uint64_t value from <b>a</b> in network byte order. */ +uint64_t +tor_htonll(uint64_t a) +{ +#ifdef WORDS_BIGENDIAN + /* Big endian. */ + return a; +#else /* WORDS_BIGENDIAN */ + /* Little endian. The worst... */ + return htonl((uint32_t)(a>>32)) | + (((uint64_t)htonl((uint32_t)a))<<32); +#endif /* WORDS_BIGENDIAN */ +} + +/** Return a uint64_t value from <b>a</b> in host byte order. */ +uint64_t +tor_ntohll(uint64_t a) +{ + return tor_htonll(a); +} + diff --git a/src/common/util.h b/src/common/util.h index ebcf88b32d..0d48eac1ad 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -22,6 +22,7 @@ /* for the correct alias to struct stat */ #include <sys/stat.h> #endif +#include "util_bug.h" #ifndef O_BINARY #define O_BINARY 0 @@ -33,41 +34,6 @@ #define O_NOFOLLOW 0 #endif -/* Replace assert() with a variant that sends failures to the log before - * calling assert() normally. - */ -#ifdef NDEBUG -/* Nobody should ever want to build with NDEBUG set. 99% of our asserts will - * be outside the critical path anyway, so it's silly to disable bug-checking - * throughout the entire program just because a few asserts are slowing you - * down. Profile, optimize the critical path, and keep debugging on. - * - * And I'm not just saying that because some of our asserts check - * security-critical properties. - */ -#error "Sorry; we don't support building with NDEBUG." -#endif - -/* Sometimes we don't want to use assertions during branch coverage tests; it - * leads to tons of unreached branches which in reality are only assertions we - * didn't hit. */ -#if defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_TESTS) -#define tor_assert(a) STMT_BEGIN \ - (void)(a); \ - STMT_END -#else -/** Like assert(3), but send assertion failures to the log as well as to - * stderr. */ -#define tor_assert(expr) STMT_BEGIN \ - if (PREDICT_UNLIKELY(!(expr))) { \ - tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__, #expr); \ - abort(); \ - } STMT_END -#endif - -void tor_assertion_failed_(const char *fname, unsigned int line, - const char *func, const char *expr); - /* If we're building with dmalloc, we want all of our memory allocation * functions to take an extra file/line pair of arguments. If not, not. * We define DMALLOC_PARAMS to the extra parameters to insert in the @@ -81,11 +47,6 @@ void tor_assertion_failed_(const char *fname, unsigned int line, #define DMALLOC_ARGS #endif -/** Define this if you want Tor to crash when any problem comes up, - * so you can get a coredump and track things down. */ -// #define tor_fragile_assert() tor_assert(0) -#define tor_fragile_assert() - /* Memory management */ void *tor_malloc_(size_t size DMALLOC_PARAMS) ATTR_MALLOC; void *tor_malloc_zero_(size_t size DMALLOC_PARAMS) ATTR_MALLOC; @@ -100,6 +61,8 @@ void *tor_memdup_(const void *mem, size_t len DMALLOC_PARAMS) void *tor_memdup_nulterm_(const void *mem, size_t len DMALLOC_PARAMS) ATTR_MALLOC ATTR_NONNULL((1)); void tor_free_(void *mem); +uint64_t tor_htonll(uint64_t a); +uint64_t tor_ntohll(uint64_t a); #ifdef USE_DMALLOC extern int dmalloc_free(const char *file, const int line, void *pnt, const int func_id); @@ -184,7 +147,6 @@ uint64_t round_to_power_of_2(uint64_t u64); unsigned round_to_next_multiple_of(unsigned number, unsigned divisor); uint32_t round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor); uint64_t round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor); -int64_t round_int64_to_next_multiple_of(int64_t number, int64_t divisor); int64_t sample_laplace_distribution(double mu, double b, double p); int64_t add_laplace_noise(int64_t signal, double random, double delta_f, double epsilon); @@ -349,6 +311,8 @@ const char *stream_status_to_string(enum stream_status stream_status); enum stream_status get_string_from_pipe(FILE *stream, char *buf, size_t count); +MOCK_DECL(int,tor_unlink,(const char *pathname)); + /** Return values from file_status(); see that function's documentation * for details. */ typedef enum { FN_ERROR, FN_NOENT, FN_FILE, FN_DIR, FN_EMPTY } file_status_t; @@ -364,8 +328,9 @@ typedef unsigned int cpd_check_t; #define CPD_GROUP_READ (1u << 3) #define CPD_CHECK_MODE_ONLY (1u << 4) #define CPD_RELAX_DIRMODE_CHECK (1u << 5) -int check_private_dir(const char *dirname, cpd_check_t check, - const char *effective_user); +MOCK_DECL(int, check_private_dir, + (const char *dirname, cpd_check_t check, + const char *effective_user)); #define OPEN_FLAGS_REPLACE (O_WRONLY|O_CREAT|O_TRUNC) #define OPEN_FLAGS_APPEND (O_WRONLY|O_CREAT|O_APPEND) @@ -378,7 +343,8 @@ FILE *start_writing_to_stdio_file(const char *fname, int open_flags, int mode, FILE *fdopen_file(open_file_t *file_data); int finish_writing_to_file(open_file_t *file_data); int abort_writing_to_file(open_file_t *file_data); -int write_str_to_file(const char *fname, const char *str, int bin); +MOCK_DECL(int, +write_str_to_file,(const char *fname, const char *str, int bin)); MOCK_DECL(int, write_bytes_to_file,(const char *fname, const char *str, size_t len, int bin)); @@ -403,18 +369,17 @@ int write_bytes_to_new_file(const char *fname, const char *str, size_t len, #ifndef _WIN32 struct stat; #endif -char *read_file_to_str(const char *filename, int flags, struct stat *stat_out) - ATTR_MALLOC; +MOCK_DECL_ATTR(char *, read_file_to_str, + (const char *filename, int flags, struct stat *stat_out), + ATTR_MALLOC); char *read_file_to_str_until_eof(int fd, size_t max_bytes_to_read, size_t *sz_out) ATTR_MALLOC; const char *parse_config_line_from_str_verbose(const char *line, char **key_out, char **value_out, const char **err_out); -#define parse_config_line_from_str(line,key_out,value_out) \ - parse_config_line_from_str_verbose((line),(key_out),(value_out),NULL) char *expand_filename(const char *filename); -struct smartlist_t *tor_listdir(const char *dirname); +MOCK_DECL(struct smartlist_t *, tor_listdir, (const char *dirname)); int path_is_relative(const char *filename); /* Process helpers */ @@ -575,6 +540,10 @@ STATIC int format_helper_exit_status(unsigned char child_state, #endif +#ifdef TOR_UNIT_TESTS +int size_mul_check__(const size_t x, const size_t y); +#endif + #define ARRAY_LENGTH(x) ((sizeof(x)) / sizeof(x[0])) #endif diff --git a/src/common/util_bug.c b/src/common/util_bug.c new file mode 100644 index 0000000000..e3e1d6df90 --- /dev/null +++ b/src/common/util_bug.c @@ -0,0 +1,53 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file util_bug.c + **/ + +#include "orconfig.h" +#include "util_bug.h" +#include "torlog.h" +#include "backtrace.h" + +/** Helper for tor_assert: report the assertion failure. */ +void +tor_assertion_failed_(const char *fname, unsigned int line, + const char *func, const char *expr) +{ + char buf[256]; + log_err(LD_BUG, "%s:%u: %s: Assertion %s failed; aborting.", + fname, line, func, expr); + tor_snprintf(buf, sizeof(buf), + "Assertion %s failed in %s at %s:%u", + expr, func, fname, line); + log_backtrace(LOG_ERR, LD_BUG, buf); +} + +/** Helper for tor_assert_nonfatal: report the assertion failure. */ +void +tor_bug_occurred_(const char *fname, unsigned int line, + const char *func, const char *expr, + int once) +{ + char buf[256]; + const char *once_str = once ? + " (Future instances of this warning will be silenced.)": ""; + if (! expr) { + log_warn(LD_BUG, "%s:%u: %s: This line should not have been reached.%s", + fname, line, func, once_str); + tor_snprintf(buf, sizeof(buf), + "Line unexpectedly reached at %s at %s:%u", + func, fname, line); + } else { + log_warn(LD_BUG, "%s:%u: %s: Non-fatal assertion %s failed.%s", + fname, line, func, expr, once_str); + tor_snprintf(buf, sizeof(buf), + "Non-fatal assertion %s failed in %s at %s:%u", + expr, func, fname, line); + } + log_backtrace(LOG_WARN, LD_BUG, buf); +} + diff --git a/src/common/util_bug.h b/src/common/util_bug.h new file mode 100644 index 0000000000..3f77e0a99e --- /dev/null +++ b/src/common/util_bug.h @@ -0,0 +1,150 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file util_bug.h + **/ + +#ifndef TOR_UTIL_BUG_H +#define TOR_UTIL_BUG_H + +#include "orconfig.h" +#include "compat.h" +#include "testsupport.h" + +/* Replace assert() with a variant that sends failures to the log before + * calling assert() normally. + */ +#ifdef NDEBUG +/* Nobody should ever want to build with NDEBUG set. 99% of our asserts will + * be outside the critical path anyway, so it's silly to disable bug-checking + * throughout the entire program just because a few asserts are slowing you + * down. Profile, optimize the critical path, and keep debugging on. + * + * And I'm not just saying that because some of our asserts check + * security-critical properties. + */ +#error "Sorry; we don't support building with NDEBUG." +#endif + +/* Sometimes we don't want to use assertions during branch coverage tests; it + * leads to tons of unreached branches which in reality are only assertions we + * didn't hit. */ +#if defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_TESTS) +#define tor_assert(a) STMT_BEGIN \ + (void)(a); \ + STMT_END +#else +/** Like assert(3), but send assertion failures to the log as well as to + * stderr. */ +#define tor_assert(expr) STMT_BEGIN \ + if (PREDICT_UNLIKELY(!(expr))) { \ + tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__, #expr); \ + abort(); \ + } STMT_END +#endif + +#define tor_assert_unreached() tor_assert(0) + +/* Non-fatal bug assertions. The "unreached" variants mean "this line should + * never be reached." The "once" variants mean "Don't log a warning more than + * once". + * + * The 'BUG' macro checks a boolean condition and logs an error message if it + * is true. Example usage: + * if (BUG(x == NULL)) + * return -1; + */ + +#ifdef ALL_BUGS_ARE_FATAL +#define tor_assert_nonfatal_unreached() tor_assert(0) +#define tor_assert_nonfatal(cond) tor_assert((cond)) +#define tor_assert_nonfatal_unreached_once() tor_assert(0) +#define tor_assert_nonfatal_once(cond) tor_assert((cond)) +#define BUG(cond) \ + (PREDICT_UNLIKELY(cond) ? \ + (tor_assertion_failed_(SHORT_FILE__,__LINE__,__func__,#cond), abort(), 1) \ + : 0) +#elif defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_TESTS) +#define tor_assert_nonfatal_unreached() STMT_NIL +#define tor_assert_nonfatal(cond) ((void)(cond)) +#define tor_assert_nonfatal_unreached_once() STMT_NIL +#define tor_assert_nonfatal_once(cond) ((void)(cond)) +#define BUG(cond) (PREDICT_UNLIKELY(cond) ? 1 : 0) +#else /* Normal case, !ALL_BUGS_ARE_FATAL, !DISABLE_ASSERTS_IN_UNIT_TESTS */ +#define tor_assert_nonfatal_unreached() STMT_BEGIN \ + tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 0); \ + STMT_END +#define tor_assert_nonfatal(cond) STMT_BEGIN \ + if (PREDICT_UNLIKELY(!(cond))) { \ + tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 0); \ + } \ + STMT_END +#define tor_assert_nonfatal_unreached_once() STMT_BEGIN \ + static int warning_logged__ = 0; \ + if (!warning_logged__) { \ + warning_logged__ = 1; \ + tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 1); \ + } \ + STMT_END +#define tor_assert_nonfatal_once(cond) STMT_BEGIN \ + static int warning_logged__ = 0; \ + if (!warning_logged__ && PREDICT_UNLIKELY(!(cond))) { \ + warning_logged__ = 1; \ + tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 1); \ + } \ + STMT_END +#define BUG(cond) \ + (PREDICT_UNLIKELY(cond) ? \ + (tor_bug_occurred_(SHORT_FILE__,__LINE__,__func__,#cond,0), 1) \ + : 0) +#endif + +#ifdef __GNUC__ +#define IF_BUG_ONCE__(cond,var) \ + if (( { \ + static int var = 0; \ + int bool_result = (cond); \ + if (PREDICT_UNLIKELY(bool_result) && !var) { \ + var = 1; \ + tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 1); \ + } \ + PREDICT_UNLIKELY(bool_result); } )) +#else +#define IF_BUG_ONCE__(cond,var) \ + static int var = 0; \ + if (PREDICT_UNLIKELY(cond)) ? \ + (var ? 1 : \ + (var=1, \ + tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 1), \ + 1)) \ + : 0) +#endif +#define IF_BUG_ONCE_VARNAME_(a) \ + warning_logged_on_ ## a ## __ +#define IF_BUG_ONCE_VARNAME__(a) \ + IF_BUG_ONCE_VARNAME_(a) + +/** This macro behaves as 'if (bug(x))', except that it only logs its + * warning once, no matter how many times it triggers. + */ + +#define IF_BUG_ONCE(cond) \ + IF_BUG_ONCE__((cond), \ + IF_BUG_ONCE_VARNAME__(__LINE__)) + +/** Define this if you want Tor to crash when any problem comes up, + * so you can get a coredump and track things down. */ +// #define tor_fragile_assert() tor_assert_unreached(0) +#define tor_fragile_assert() tor_assert_nonfatal_unreached_once() + +void tor_assertion_failed_(const char *fname, unsigned int line, + const char *func, const char *expr); +void tor_bug_occurred_(const char *fname, unsigned int line, + const char *func, const char *expr, + int once); + +#endif + diff --git a/src/common/util_format.c b/src/common/util_format.c index 8aae9e8771..9009e1a814 100644 --- a/src/common/util_format.c +++ b/src/common/util_format.c @@ -21,33 +21,48 @@ #include <string.h> #include <stdlib.h> -/** Implements base32 encoding as in RFC 4648. Limitation: Requires - * that srclen*8 is a multiple of 5. - */ +/* Return the base32 encoded size in bytes using the source length srclen. + * The NUL terminated byte is added as well since every base32 encoding + * requires enough space for it. */ +size_t +base32_encoded_size(size_t srclen) +{ + size_t enclen; + enclen = CEIL_DIV(srclen*8, 5) + 1; + tor_assert(enclen < INT_MAX && enclen > srclen); + return enclen; +} + +/** Implements base32 encoding as in RFC 4648. */ void base32_encode(char *dest, size_t destlen, const char *src, size_t srclen) { unsigned int i, v, u; - size_t nbits = srclen * 8, bit; + size_t nbits = srclen * 8; + size_t bit; tor_assert(srclen < SIZE_T_CEILING/8); - tor_assert((nbits%5) == 0); /* We need an even multiple of 5 bits. */ - tor_assert((nbits/5)+1 <= destlen); /* We need enough space. */ + /* We need enough space for the encoded data and the extra NUL byte. */ + tor_assert(base32_encoded_size(srclen) <= destlen); tor_assert(destlen < SIZE_T_CEILING); + /* Make sure we leave no uninitialized data in the destination buffer. */ + memset(dest, 0, destlen); + for (i=0,bit=0; bit < nbits; ++i, bit+=5) { /* set v to the 16-bit value starting at src[bits/8], 0-padded. */ v = ((uint8_t)src[bit/8]) << 8; - if (bit+5<nbits) v += (uint8_t)src[(bit/8)+1]; - /* set u to the 5-bit value at the bit'th bit of src. */ + if (bit+5<nbits) + v += (uint8_t)src[(bit/8)+1]; + /* set u to the 5-bit value at the bit'th bit of buf. */ u = (v >> (11-(bit%8))) & 0x1F; dest[i] = BASE32_CHARS[u]; } dest[i] = '\0'; } -/** Implements base32 decoding as in RFC 4648. Limitation: Requires - * that srclen*5 is a multiple of 8. Returns 0 if successful, -1 otherwise. +/** Implements base32 decoding as in RFC 4648. + * Returns 0 if successful, -1 otherwise. */ int base32_decode(char *dest, size_t destlen, const char *src, size_t srclen) @@ -57,13 +72,13 @@ base32_decode(char *dest, size_t destlen, const char *src, size_t srclen) unsigned int i; size_t nbits, j, bit; char *tmp; - nbits = srclen * 5; + nbits = ((srclen * 5) / 8) * 8; tor_assert(srclen < SIZE_T_CEILING / 5); - tor_assert((nbits%8) == 0); /* We need an even multiple of 8 bits. */ tor_assert((nbits/8) <= destlen); /* We need enough space. */ tor_assert(destlen < SIZE_T_CEILING); + /* Make sure we leave no uninitialized data in the destination buffer. */ memset(dest, 0, destlen); /* Convert base32 encoded chars to the 5-bit values that they represent. */ @@ -186,7 +201,8 @@ base64_encode(char *dest, size_t destlen, const char *src, size_t srclen, if (enclen > INT_MAX) return -1; - memset(dest, 0, enclen); + /* Make sure we leave no uninitialized data in the destination buffer. */ + memset(dest, 0, destlen); /* XXX/Yawning: If this ends up being too slow, this can be sped up * by separating the multiline format case and the normal case, and @@ -249,7 +265,7 @@ base64_encode(char *dest, size_t destlen, const char *src, size_t srclen, break; default: /* Something went catastrophically wrong. */ - tor_fragile_assert(); + tor_fragile_assert(); // LCOV_EXCL_LINE return -1; } @@ -387,6 +403,7 @@ base64_decode(char *dest, size_t destlen, const char *src, size_t srclen) if (destlen > SIZE_T_CEILING) return -1; + /* Make sure we leave no uninitialized data in the destination buffer. */ memset(dest, 0, destlen); /* Iterate over all the bytes in src. Each one will add 0 or 6 bits to the @@ -461,6 +478,9 @@ base16_encode(char *dest, size_t destlen, const char *src, size_t srclen) tor_assert(destlen >= srclen*2+1); tor_assert(destlen < SIZE_T_CEILING); + /* Make sure we leave no uninitialized data in the destination buffer. */ + memset(dest, 0, destlen); + cp = dest; end = src+srclen; while (src<end) { @@ -504,20 +524,24 @@ hex_decode_digit(char c) return hex_decode_digit_(c); } -/** Given a hexadecimal string of <b>srclen</b> bytes in <b>src</b>, decode it - * and store the result in the <b>destlen</b>-byte buffer at <b>dest</b>. - * Return 0 on success, -1 on failure. */ +/** Given a hexadecimal string of <b>srclen</b> bytes in <b>src</b>, decode + * it and store the result in the <b>destlen</b>-byte buffer at <b>dest</b>. + * Return the number of bytes decoded on success, -1 on failure. If + * <b>destlen</b> is greater than INT_MAX or less than half of + * <b>srclen</b>, -1 is returned. */ int base16_decode(char *dest, size_t destlen, const char *src, size_t srclen) { const char *end; - + char *dest_orig = dest; int v1,v2; + if ((srclen % 2) != 0) return -1; - if (destlen < srclen/2 || destlen > SIZE_T_CEILING) + if (destlen < srclen/2 || destlen > INT_MAX) return -1; + /* Make sure we leave no uninitialized data in the destination buffer. */ memset(dest, 0, destlen); end = src+srclen; @@ -530,6 +554,9 @@ base16_decode(char *dest, size_t destlen, const char *src, size_t srclen) ++dest; src+=2; } - return 0; + + tor_assert((dest-dest_orig) <= (ptrdiff_t) destlen); + + return (int) (dest-dest_orig); } diff --git a/src/common/util_format.h b/src/common/util_format.h index a748a4f3cf..20ac711d10 100644 --- a/src/common/util_format.h +++ b/src/common/util_format.h @@ -24,6 +24,7 @@ int base64_decode_nopad(uint8_t *dest, size_t destlen, #define BASE32_CHARS "abcdefghijklmnopqrstuvwxyz234567" void base32_encode(char *dest, size_t destlen, const char *src, size_t srclen); int base32_decode(char *dest, size_t destlen, const char *src, size_t srclen); +size_t base32_encoded_size(size_t srclen); int hex_decode_digit(char c); void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen); diff --git a/src/common/util_process.c b/src/common/util_process.c index 848b238318..abda63720c 100644 --- a/src/common/util_process.c +++ b/src/common/util_process.c @@ -61,9 +61,9 @@ process_map_entries_eq_(const waitpid_callback_t *a, static HT_HEAD(process_map, waitpid_callback_t) process_map = HT_INITIALIZER(); HT_PROTOTYPE(process_map, waitpid_callback_t, node, process_map_entry_hash_, - process_map_entries_eq_); + process_map_entries_eq_) HT_GENERATE2(process_map, waitpid_callback_t, node, process_map_entry_hash_, - process_map_entries_eq_, 0.6, tor_reallocarray_, tor_free_); + process_map_entries_eq_, 0.6, tor_reallocarray_, tor_free_) /** * Begin monitoring the child pid <b>pid</b> to see if we get a SIGCHLD for diff --git a/src/common/workqueue.c b/src/common/workqueue.c index 0a38550de0..48c0cca01f 100644 --- a/src/common/workqueue.c +++ b/src/common/workqueue.c @@ -262,9 +262,12 @@ workerthread_new(void *state, threadpool_t *pool, replyqueue_t *replyqueue) thr->in_pool = pool; if (spawn_func(worker_thread_main, thr) < 0) { + //LCOV_EXCL_START + tor_assert_nonfatal_unreached(); log_err(LD_GENERAL, "Can't launch worker thread."); tor_free(thr); return NULL; + //LCOV_EXCL_STOP } return thr; @@ -375,8 +378,8 @@ threadpool_queue_update(threadpool_t *pool, static int threadpool_start_threads(threadpool_t *pool, int n) { - if (n < 0) - return -1; + if (BUG(n < 0)) + return -1; // LCOV_EXCL_LINE if (n > MAX_THREADS) n = MAX_THREADS; @@ -391,9 +394,12 @@ threadpool_start_threads(threadpool_t *pool, int n) workerthread_t *thr = workerthread_new(state, pool, pool->reply_queue); if (!thr) { + //LCOV_EXCL_START + tor_assert_nonfatal_unreached(); pool->free_thread_state_fn(state); tor_mutex_release(&pool->lock); return -1; + //LCOV_EXCL_STOP } thr->index = pool->n_threads; pool->threads[pool->n_threads++] = thr; @@ -429,10 +435,13 @@ threadpool_new(int n_threads, pool->reply_queue = replyqueue; if (threadpool_start_threads(pool, n_threads) < 0) { + //LCOV_EXCL_START + tor_assert_nonfatal_unreached(); tor_cond_uninit(&pool->condition); tor_mutex_uninit(&pool->lock); tor_free(pool); return NULL; + //LCOV_EXCL_STOP } return pool; @@ -456,8 +465,10 @@ replyqueue_new(uint32_t alertsocks_flags) rq = tor_malloc_zero(sizeof(replyqueue_t)); if (alert_sockets_create(&rq->alert, alertsocks_flags) < 0) { + //LCOV_EXCL_START tor_free(rq); return NULL; + //LCOV_EXCL_STOP } tor_mutex_init(&rq->lock); @@ -486,10 +497,12 @@ void replyqueue_process(replyqueue_t *queue) { if (queue->alert.drain_fn(queue->alert.read_fd) < 0) { + //LCOV_EXCL_START static ratelim_t warn_limit = RATELIM_INIT(7200); log_fn_ratelim(&warn_limit, LOG_WARN, LD_GENERAL, "Failure from drain_fd: %s", tor_socket_strerror(tor_socket_errno(queue->alert.read_fd))); + //LCOV_EXCL_STOP } tor_mutex_acquire(&queue->lock); diff --git a/src/common/workqueue.h b/src/common/workqueue.h index 89282e6f21..54276767b0 100644 --- a/src/common/workqueue.h +++ b/src/common/workqueue.h @@ -7,7 +7,7 @@ #include "compat.h" /** A replyqueue is used to tell the main thread about the outcome of - * work that we queued for the the workers. */ + * work that we queued for the workers. */ typedef struct replyqueue_s replyqueue_t; /** A thread-pool manages starting threads and passing work to them. */ typedef struct threadpool_s threadpool_t; diff --git a/src/config/torrc.minimal.in-staging b/src/config/torrc.minimal.in-staging index 248cb5cf02..d4dfd5f6bb 100644 --- a/src/config/torrc.minimal.in-staging +++ b/src/config/torrc.minimal.in-staging @@ -98,6 +98,8 @@ # OutboundBindAddress 10.0.0.5 ## A handle for your relay, so people don't have to refer to it by key. +## Nicknames must be between 1 and 19 characters inclusive, and must +## contain only the characters [a-zA-Z0-9]. #Nickname ididnteditheconfig ## Define these to limit how much relayed traffic you will allow. Your diff --git a/src/config/torrc.sample.in b/src/config/torrc.sample.in index 248cb5cf02..d4dfd5f6bb 100644 --- a/src/config/torrc.sample.in +++ b/src/config/torrc.sample.in @@ -98,6 +98,8 @@ # OutboundBindAddress 10.0.0.5 ## A handle for your relay, so people don't have to refer to it by key. +## Nicknames must be between 1 and 19 characters inclusive, and must +## contain only the characters [a-zA-Z0-9]. #Nickname ididnteditheconfig ## Define these to limit how much relayed traffic you will allow. Your diff --git a/src/ext/README b/src/ext/README index 7ce1bc3b74..dfe620ed16 100644 --- a/src/ext/README +++ b/src/ext/README @@ -73,3 +73,14 @@ readpassphrase.[ch] Portable readpassphrase implementation from OpenSSH portable, version 6.8p1. + +timeouts/ + + William Ahern's hierarchical timer-wheel implementation. MIT license. + +mulodi/ + + Contains an overflow-checking 64-bit signed integer multiply + from LLVM's compiler_rt. For some reason, this is missing from + 32-bit libclang in many places. Dual licensed MIT-license and + BSD-like license; see mulodi/LICENSE.TXT. diff --git a/src/ext/ed25519/donna/curve25519-donna-64bit.h b/src/ext/ed25519/donna/curve25519-donna-64bit.h index 2941d1bcdc..50c9916768 100644 --- a/src/ext/ed25519/donna/curve25519-donna-64bit.h +++ b/src/ext/ed25519/donna/curve25519-donna-64bit.h @@ -8,9 +8,9 @@ typedef uint64_t bignum25519[5]; -static const uint64_t reduce_mask_40 = ((uint64_t)1 << 40) - 1; +//static const uint64_t reduce_mask_40 = ((uint64_t)1 << 40) - 1; static const uint64_t reduce_mask_51 = ((uint64_t)1 << 51) - 1; -static const uint64_t reduce_mask_56 = ((uint64_t)1 << 56) - 1; +//static const uint64_t reduce_mask_56 = ((uint64_t)1 << 56) - 1; /* out = in */ DONNA_INLINE static void diff --git a/src/ext/ed25519/donna/ed25519-donna-64bit-x86.h b/src/ext/ed25519/donna/ed25519-donna-64bit-x86.h index 30bd472762..f6b5570298 100644 --- a/src/ext/ed25519/donna/ed25519-donna-64bit-x86.h +++ b/src/ext/ed25519/donna/ed25519-donna-64bit-x86.h @@ -2,6 +2,11 @@ #define HAVE_GE25519_SCALARMULT_BASE_CHOOSE_NIELS +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Woverlength-strings" +#endif + DONNA_NOINLINE static void ge25519_scalarmult_base_choose_niels(ge25519_niels *t, const uint8_t table[256][96], uint32_t pos, signed char b) { int64_t breg = (int64_t)b; @@ -347,5 +352,9 @@ ge25519_scalarmult_base_choose_niels(ge25519_niels *t, const uint8_t table[256][ ); } +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + #endif /* defined(ED25519_GCC_64BIT_X86_CHOOSE) */ diff --git a/src/ext/ed25519/donna/ed25519-donna-batchverify.h b/src/ext/ed25519/donna/ed25519-donna-batchverify.h index 43c4923b3e..7c64cce787 100644 --- a/src/ext/ed25519/donna/ed25519-donna-batchverify.h +++ b/src/ext/ed25519/donna/ed25519-donna-batchverify.h @@ -188,7 +188,7 @@ ge25519_multi_scalarmult_vartime(ge25519 *r, batch_heap *heap, size_t count) { } /* not actually used for anything other than testing */ -unsigned char batch_point_buffer[3][32]; +static unsigned char batch_point_buffer[3][32]; static int ge25519_is_neutral_vartime(const ge25519 *p) { diff --git a/src/ext/ed25519/donna/ed25519-donna.h b/src/ext/ed25519/donna/ed25519-donna.h index 64561d3288..299c8d90fd 100644 --- a/src/ext/ed25519/donna/ed25519-donna.h +++ b/src/ext/ed25519/donna/ed25519-donna.h @@ -10,6 +10,16 @@ #include "ed25519-donna-portable.h" +#include "orconfig.h" + +#ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS +/* Some of the ASM here is very long strings. */ +#ifdef __clang__ +#pragma clang diagnostic ignored "-Woverlength-strings" +#else +#pragma GCC diagnostic ignored "-Woverlength-strings" +#endif +#endif #if defined(ED25519_SSE2) #else diff --git a/src/ext/ed25519/donna/ed25519_tor.c b/src/ext/ed25519/donna/ed25519_tor.c index 52b259dfe1..07f6a0f23a 100644 --- a/src/ext/ed25519/donna/ed25519_tor.c +++ b/src/ext/ed25519/donna/ed25519_tor.c @@ -44,7 +44,8 @@ typedef unsigned char ed25519_signature[64]; typedef unsigned char ed25519_public_key[32]; typedef unsigned char ed25519_secret_key[32]; -static void gettweak(unsigned char *out, const unsigned char *param); +static void ed25519_donna_gettweak(unsigned char *out, + const unsigned char *param); static int ED25519_FN(ed25519_sign_open) (const unsigned char *m, size_t mlen, const ed25519_public_key pk, const ed25519_signature RS); @@ -242,7 +243,7 @@ ed25519_donna_sign(unsigned char *sig, const unsigned char *m, size_t mlen, } static void -gettweak(unsigned char *out, const unsigned char *param) +ed25519_donna_gettweak(unsigned char *out, const unsigned char *param) { static const char str[] = "Derive temporary signing key"; ed25519_hash_context ctx; @@ -266,7 +267,7 @@ ed25519_donna_blind_secret_key(unsigned char *out, const unsigned char *inp, ed25519_hash_context ctx; bignum256modm ALIGN(16) sk, t; - gettweak(tweak, param); + ed25519_donna_gettweak(tweak, param); expand256_modm(t, tweak, 32); expand256_modm(sk, inp, 32); @@ -297,7 +298,7 @@ ed25519_donna_blind_public_key(unsigned char *out, const unsigned char *inp, ge25519 ALIGN(16) A, Aprime; bignum256modm ALIGN(16) t; - gettweak(tweak, param); + ed25519_donna_gettweak(tweak, param); expand256_modm(t, tweak, 32); /* No "ge25519_unpack", negate the public key. */ diff --git a/src/ext/ed25519/ref10/blinding.c b/src/ext/ed25519/ref10/blinding.c index 4d9a9cbbe7..ee3e8666fa 100644 --- a/src/ext/ed25519/ref10/blinding.c +++ b/src/ext/ed25519/ref10/blinding.c @@ -10,7 +10,7 @@ #include "crypto.h" static void -gettweak(unsigned char *out, const unsigned char *param) +ed25519_ref10_gettweak(unsigned char *out, const unsigned char *param) { const char str[] = "Derive temporary signing key"; crypto_hash_sha512_2(out, (const unsigned char*)str, strlen(str), param, 32); @@ -26,7 +26,7 @@ int ed25519_ref10_blind_secret_key(unsigned char *out, const char str[] = "Derive temporary signing key hash input"; unsigned char tweak[64]; unsigned char zero[32]; - gettweak(tweak, param); + ed25519_ref10_gettweak(tweak, param); memset(zero, 0, 32); sc_muladd(out, inp, tweak, zero); @@ -50,7 +50,7 @@ int ed25519_ref10_blind_public_key(unsigned char *out, ge_p3 A; ge_p2 Aprime; - gettweak(tweak, param); + ed25519_ref10_gettweak(tweak, param); memset(zero, 0, sizeof(zero)); /* Not the greatest implementation of all of this. I wish I had diff --git a/src/ext/eventdns.c b/src/ext/eventdns.c index fc5657cbb4..f5b7723b54 100644 --- a/src/ext/eventdns.c +++ b/src/ext/eventdns.c @@ -50,9 +50,6 @@ #endif #endif -/* #define _POSIX_C_SOURCE 200507 */ -#define _GNU_SOURCE - #ifdef DNS_USE_CPU_CLOCK_FOR_ID #ifdef DNS_USE_OPENSSL_FOR_ID #error Multiple id options selected @@ -2004,8 +2001,7 @@ evdns_request_timeout_callback(int fd, short events, void *arg) { } else { /* retransmit it */ /* Stop waiting for the timeout. No need to do this in - * request_finished; that one already deletes the timeout event. - * XXXX023 port this change to libevent. */ + * request_finished; that one already deletes the timeout event. */ del_timeout_event(req); evdns_request_transmit(req); } diff --git a/src/ext/ht.h b/src/ext/ht.h index 28d1fe49d5..1b6cbe6632 100644 --- a/src/ext/ht.h +++ b/src/ext/ht.h @@ -203,6 +203,7 @@ ht_string_hash(const char *s) name##_HT_GROW(head, head->hth_n_entries+1); \ HT_SET_HASH_(elm, field, hashfn); \ p = name##_HT_FIND_P_(head, elm); \ + HT_ASSERT_(p != NULL); /* this holds because we called HT_GROW */ \ r = *p; \ *p = elm; \ if (r && (r!=elm)) { \ @@ -470,6 +471,7 @@ ht_string_hash(const char *s) name##_HT_GROW(var##_head_, var##_head_->hth_n_entries+1); \ HT_SET_HASH_((elm), field, hashfn); \ var = name##_HT_FIND_P_(var##_head_, (elm)); \ + HT_ASSERT_(var); /* Holds because we called HT_GROW */ \ if (*var) { \ y; \ } else { \ diff --git a/src/ext/include.am b/src/ext/include.am index bf678f2c9d..6cfdbcc447 100644 --- a/src/ext/include.am +++ b/src/ext/include.am @@ -12,11 +12,16 @@ EXTHEADERS = \ src/ext/strlcpy.c \ src/ext/tinytest_macros.h \ src/ext/tor_queue.h \ - src/ext/siphash.h + src/ext/siphash.h \ + src/ext/timeouts/timeout.h \ + src/ext/timeouts/timeout-debug.h \ + src/ext/timeouts/timeout-bitops.c \ + src/ext/timeouts/timeout.c noinst_HEADERS+= $(EXTHEADERS) -src_ext_ed25519_ref10_libed25519_ref10_a_CFLAGS= +src_ext_ed25519_ref10_libed25519_ref10_a_CFLAGS=\ + @CFLAGS_CONSTTIME@ src_ext_ed25519_ref10_libed25519_ref10_a_SOURCES= \ src/ext/ed25519/ref10/fe_0.c \ @@ -93,7 +98,8 @@ noinst_HEADERS += $(ED25519_REF10_HDRS) LIBED25519_REF10=src/ext/ed25519/ref10/libed25519_ref10.a noinst_LIBRARIES += $(LIBED25519_REF10) -src_ext_ed25519_donna_libed25519_donna_a_CFLAGS= \ +src_ext_ed25519_donna_libed25519_donna_a_CFLAGS=\ + @CFLAGS_CONSTTIME@ \ -DED25519_CUSTOMRANDOM \ -DED25519_SUFFIX=_donna @@ -135,7 +141,8 @@ noinst_HEADERS += $(ED25519_DONNA_HDRS) LIBED25519_DONNA=src/ext/ed25519/donna/libed25519_donna.a noinst_LIBRARIES += $(LIBED25519_DONNA) -src_ext_keccak_tiny_libkeccak_tiny_a_CFLAGS= +src_ext_keccak_tiny_libkeccak_tiny_a_CFLAGS=\ + @CFLAGS_CONSTTIME@ src_ext_keccak_tiny_libkeccak_tiny_a_SOURCES= \ src/ext/keccak-tiny/keccak-tiny-unrolled.c @@ -148,3 +155,21 @@ noinst_HEADERS += $(LIBKECCAK_TINY_HDRS) LIBKECCAK_TINY=src/ext/keccak-tiny/libkeccak-tiny.a noinst_LIBRARIES += $(LIBKECCAK_TINY) +EXTRA_DIST += \ + src/ext/timeouts/bench/bench-add.lua \ + src/ext/timeouts/bench/bench-aux.lua \ + src/ext/timeouts/bench/bench.c \ + src/ext/timeouts/bench/bench-del.lua \ + src/ext/timeouts/bench/bench-expire.lua \ + src/ext/timeouts/bench/bench.h \ + src/ext/timeouts/bench/bench-heap.c \ + src/ext/timeouts/bench/bench-llrb.c \ + src/ext/timeouts/bench/bench.plt \ + src/ext/timeouts/bench/bench-wheel.c \ + src/ext/timeouts/bench/Rules.mk \ + src/ext/timeouts/lua/Rules.mk \ + src/ext/timeouts/lua/timeout-lua.c \ + src/ext/timeouts/Makefile \ + src/ext/timeouts/Rules.shrc \ + src/ext/timeouts/test-timeout.c + diff --git a/src/ext/mulodi/LICENSE.TXT b/src/ext/mulodi/LICENSE.TXT new file mode 100644 index 0000000000..a17dc12b27 --- /dev/null +++ b/src/ext/mulodi/LICENSE.TXT @@ -0,0 +1,91 @@ +============================================================================== +compiler_rt License +============================================================================== + +The compiler_rt library is dual licensed under both the University of Illinois +"BSD-Like" license and the MIT license. As a user of this code you may choose +to use it under either license. As a contributor, you agree to allow your code +to be used under both. + +Full text of the relevant licenses is included below. + +============================================================================== + +University of Illinois/NCSA +Open Source License + +Copyright (c) 2009-2016 by the contributors listed in CREDITS.TXT + +All rights reserved. + +Developed by: + + LLVM Team + + University of Illinois at Urbana-Champaign + + http://llvm.org + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal with +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + * Neither the names of the LLVM Team, University of Illinois at + Urbana-Champaign, nor the names of its contributors may be used to + endorse or promote products derived from this Software without specific + prior written permission. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. + +============================================================================== + +Copyright (c) 2009-2015 by the contributors listed in CREDITS.TXT + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +============================================================================== +Copyrights and Licenses for Third Party Software Distributed with LLVM: +============================================================================== +The LLVM software contains code written by third parties. Such software will +have its own individual LICENSE.TXT file in the directory in which it appears. +This file will describe the copyrights, license, and restrictions which apply +to that code. + +The disclaimer of warranty in the University of Illinois Open Source License +applies to all code in the LLVM Distribution, and nothing in any of the +other licenses gives permission to use the names of the LLVM Team or the +University of Illinois to endorse or promote products derived from this +Software. + diff --git a/src/ext/mulodi/mulodi4.c b/src/ext/mulodi/mulodi4.c new file mode 100644 index 0000000000..bfa5e01295 --- /dev/null +++ b/src/ext/mulodi/mulodi4.c @@ -0,0 +1,66 @@ +/*===-- mulodi4.c - Implement __mulodi4 -----------------------------------=== + * + * The LLVM Compiler Infrastructure + * + * This file is dual licensed under the MIT and the University of Illinois Open + * Source Licenses. See LICENSE.TXT for details. + * + * ===----------------------------------------------------------------------=== + * + * This file implements __mulodi4 for the compiler_rt library. + * + * ===----------------------------------------------------------------------=== + */ + +#if 0 +#include "int_lib.h" +#else +#define COMPILER_RT_ABI +#define di_int int64_t +#include "torint.h" + +di_int __mulodi4(di_int a, di_int b, int* overflow); +#endif + +/* Returns: a * b */ + +/* Effects: sets *overflow to 1 if a * b overflows */ + +COMPILER_RT_ABI di_int +__mulodi4(di_int a, di_int b, int* overflow) +{ + const int N = (int)(sizeof(di_int) * CHAR_BIT); + const di_int MIN = (di_int)1 << (N-1); + const di_int MAX = ~MIN; + *overflow = 0; + di_int result = a * b; + if (a == MIN) + { + if (b != 0 && b != 1) + *overflow = 1; + return result; + } + if (b == MIN) + { + if (a != 0 && a != 1) + *overflow = 1; + return result; + } + di_int sa = a >> (N - 1); + di_int abs_a = (a ^ sa) - sa; + di_int sb = b >> (N - 1); + di_int abs_b = (b ^ sb) - sb; + if (abs_a < 2 || abs_b < 2) + return result; + if (sa == sb) + { + if (abs_a > MAX / abs_b) + *overflow = 1; + } + else + { + if (abs_a > MIN / -abs_b) + *overflow = 1; + } + return result; +} diff --git a/src/ext/timeouts/Makefile b/src/ext/timeouts/Makefile new file mode 100644 index 0000000000..554ebb9ddd --- /dev/null +++ b/src/ext/timeouts/Makefile @@ -0,0 +1,68 @@ +# NOTE: GNU Make 3.81 won't export MAKEFLAGS if .POSIX is specified, but +# Solaris make won't export MAKEFLAGS unless .POSIX is specified. +$(firstword ignore).POSIX: + +.DEFAULT_GOAL = all + +.SUFFIXES: + +all: + +# +# USER-MODIFIABLE MACROS +# +top_srcdir = . +top_builddir = . + +CFLAGS = -O2 -march=native -g -Wall -Wextra -Wno-unused-parameter -Wno-unused-function +SOFLAGS = $$(auto_soflags) +LIBS = $$(auto_libs) + +ALL_CPPFLAGS = -I$(top_srcdir) -DWHEEL_BIT=$(WHEEL_BIT) -DWHEEL_NUM=$(WHEEL_NUM) $(CPPFLAGS) +ALL_CFLAGS = $(CFLAGS) +ALL_SOFLAGS = $(SOFLAGS) +ALL_LDFLAGS = $(LDFLAGS) +ALL_LIBS = $(LIBS) + +LUA_API = 5.3 +LUA = lua +LUA51_CPPFLAGS = $(LUA_CPPFLAGS) +LUA52_CPPFLAGS = $(LUA_CPPFLAGS) +LUA53_CPPFLAGS = $(LUA_CPPFLAGS) + +WHEEL_BIT = 6 +WHEEL_NUM = 4 + +RM = rm -f + +# END MACROS + +SHRC = \ + top_srcdir="$(top_srcdir)"; \ + top_builddir="$(top_builddir)"; \ + . "$${top_srcdir}/Rules.shrc" + +LUA_APIS = 5.1 5.2 5.3 + +include $(top_srcdir)/lua/Rules.mk +include $(top_srcdir)/bench/Rules.mk + +all: test-timeout + +timeout.o: $(top_srcdir)/timeout.c +test-timeout.o: $(top_srcdir)/test-timeout.c + +timeout.o test-timeout.o: + @$(SHRC); echo_cmd $(CC) $(ALL_CFLAGS) -c -o $@ $${top_srcdir}/$(@F:%.o=%.c) $(ALL_CPPFLAGS) + +test-timeout: timeout.o test-timeout.o + @$(SHRC); echo_cmd $(CC) $(ALL_CPPFLAGS) $(ALL_CFLAGS) -o $@ timeout.o test-timeout.o + +.PHONY: clean clean~ + +clean: + $(RM) $(top_builddir)/test-timeout $(top_builddir)/*.o + $(RM) -r $(top_builddir)/*.dSYM + +clean~: + find $(top_builddir) $(top_srcdir) -name "*~" -exec $(RM) -- {} "+" diff --git a/src/ext/timeouts/Rules.shrc b/src/ext/timeouts/Rules.shrc new file mode 100644 index 0000000000..ece75d42d4 --- /dev/null +++ b/src/ext/timeouts/Rules.shrc @@ -0,0 +1,40 @@ +# convert to absolute paths +top_srcdir="$(cd "${top_srcdir}" && pwd -L)" +top_builddir="$(cd "${top_builddir}" && pwd -L)" + +# Paths for Lua modules (benchmarks and installed modules) +export LUA_CPATH="${top_builddir}/lua/5.1/?.so;${top_builddir}/bench/?.so;;" +export LUA_PATH="${top_srcdir}/lua/?.lua;${top_srcdir}/bench/?.lua;;" +export LUA_CPATH_5_2="${top_builddir}/lua/5.2/?.so;${top_builddir}/bench/?.so;;" +export LUA_PATH_5_2="${top_srcdir}/lua/?.lua;${top_srcdir}/bench/?.lua;;" +export LUA_CPATH_5_3="${top_builddir}/lua/5.3/?.so;${top_builddir}/bench/?.so;;" +export LUA_PATH_5_3="${top_srcdir}/lua/?.lua;${top_srcdir}/bench/?.lua;;" + +# preserve stdout so we can print commands to terminal +exec 9>&1; +echo_cmd() { + printf "%s\n" "$*" >&9; + "$@"; +} + +auto_soflags() { + case "$(uname -s)" in + Darwin) + printf -- "-bundle -undefined dynamic_lookup" + ;; + *) + printf -- "-fPIC -shared" + ;; + esac +} + +auto_libs() { + case "$(uname -s)" in + Linux) + printf -- "-lrt" + ;; + *) + ;; + esac +} + diff --git a/src/ext/timeouts/bench/Rules.mk b/src/ext/timeouts/bench/Rules.mk new file mode 100644 index 0000000000..3ee72f3eff --- /dev/null +++ b/src/ext/timeouts/bench/Rules.mk @@ -0,0 +1,49 @@ +BENCH_MODS = bench.so $(BENCH_ALGOS:%=bench-%.so) +BENCH_ALGOS = wheel heap llrb +BENCH_OPS = add del expire + +$(top_builddir)/bench/bench.so: $(top_srcdir)/bench/bench.c +$(top_builddir)/bench/bench-wheel.so: $(top_srcdir)/bench/bench-wheel.c +$(top_builddir)/bench/bench-heap.so: $(top_srcdir)/bench/bench-heap.c +$(top_builddir)/bench/bench-llrb.so: $(top_srcdir)/bench/bench-llrb.c + +$(BENCH_MODS:%=$(top_builddir)/bench/%): $(top_srcdir)/timeout.h $(top_srcdir)/timeout.c $(top_srcdir)/bench/bench.h + mkdir -p $(@D) + @$(SHRC); echo_cmd $(CC) -o $@ $(top_srcdir)/bench/$(@F:%.so=%.c) $(ALL_CPPFLAGS) $(ALL_CFLAGS) $(ALL_SOFLAGS) $(ALL_LDFLAGS) $(ALL_LIBS) + +$(BENCH_OPS:%=$(top_builddir)/bench/wheel-%.dat): $(top_builddir)/bench/bench-wheel.so $(top_builddir)/bench/bench.so $(top_srcdir)/bench/bench-aux.lua +$(BENCH_OPS:%=$(top_builddir)/bench/heap-%.dat): $(top_builddir)/bench/bench-heap.so $(top_builddir)/bench/bench.so $(top_srcdir)/bench/bench-aux.lua +$(BENCH_OPS:%=$(top_builddir)/bench/llrb-%.dat): $(top_builddir)/bench/bench-llrb.so $(top_builddir)/bench/bench.so $(top_srcdir)/bench/bench-aux.lua + +$(BENCH_ALGOS:%=$(top_builddir)/bench/%-add.dat): $(top_srcdir)/bench/bench-add.lua + @$(SHRC); echo_cmd cd $(@D) && echo_cmd $(LUA) $${top_srcdir}/bench/bench-add.lua $${top_builddir}/bench/bench-$(@F:%-add.dat=%).so > $(@F).tmp + mv $@.tmp $@ + +$(BENCH_ALGOS:%=$(top_builddir)/bench/%-del.dat): $(top_srcdir)/bench/bench-del.lua + @$(SHRC); echo_cmd cd $(@D) && echo_cmd $(LUA) $${top_srcdir}/bench/bench-del.lua $${top_builddir}/bench/bench-$(@F:%-del.dat=%).so > $(@F).tmp + mv $@.tmp $@ + +$(BENCH_ALGOS:%=$(top_builddir)/bench/%-expire.dat): $(top_srcdir)/bench/bench-expire.lua + @$(SHRC); echo_cmd cd $(@D) && echo_cmd $(LUA) $${top_srcdir}/bench/bench-expire.lua $${top_builddir}/bench/bench-$(@F:%-expire.dat=%).so > $(@F).tmp + mv $@.tmp $@ + +$(top_builddir)/bench/bench.eps: \ + $(BENCH_OPS:%=$(top_builddir)/bench/wheel-%.dat) \ + $(BENCH_OPS:%=$(top_builddir)/bench/heap-%.dat) +# $(BENCH_OPS:%=$(top_builddir)/bench/llrb-%.dat) + +$(top_builddir)/bench/bench.eps: $(top_srcdir)/bench/bench.plt + @$(SHRC); echo_cmd cd $(@D) && echo_cmd gnuplot $${top_srcdir}/bench/bench.plt > $(@F).tmp + mv $@.tmp $@ + +$(top_builddir)/bench/bench.pdf: $(top_builddir)/bench/bench.eps + @$(SHRC); echo_cmd ps2pdf $${top_builddir}/bench/bench.eps $@ + +bench-mods: $(BENCH_MODS:%=$(top_builddir)/bench/%) + +bench-all: $(top_builddir)/bench/bench.pdf + +bench-clean: + $(RM) -r $(top_builddir)/bench/*.so $(top_builddir)/bench/*.dSYM + $(RM) $(top_builddir)/bench/*.dat $(top_builddir)/bench/*.tmp + $(RM) $(top_builddir)/bench/bench.{eps,pdf} diff --git a/src/ext/timeouts/bench/bench-add.lua b/src/ext/timeouts/bench/bench-add.lua new file mode 100755 index 0000000000..64a921d3de --- /dev/null +++ b/src/ext/timeouts/bench/bench-add.lua @@ -0,0 +1,30 @@ +#!/usr/bin/env lua + +local bench = require"bench" +local aux = require"bench-aux" + +local lib = ... or aux.optenv("BENCH_L", "bench-wheel.so") +local limit = tonumber(aux.optenv("BENCH_N", 1000000)) +local step = tonumber(aux.optenv("BENCH_S", limit / 100)) +local exp_step = tonumber(aux.optenv("BENCH_E", 1.0)) +local verbose = aux.toboolean(os.getenv("BENCH_V", false)) + +local B = bench.new(lib, count, nil, verbose) +local fill_count, fill_last = B:fill(limit) + +for i=0,limit,step do + local exp_elapsed, fill_elapsed, fill_rate + + -- expire all timeouts + --exp_elapsed = aux.time(B.expire, B, fill_count, fill_last * exp_step) + exp_elapsed = aux.time(B.del, B, 0, fill_count) + assert(B:empty()) + + -- add i timeouts + fill_elapsed, fill_count, fill_last = aux.time(B.fill, B, i) + assert(fill_count == i) + fill_rate = fill_elapsed > 0 and (fill_count / fill_elapsed) or 0 + + local fmt = verbose and "%d\t%f\t(%d/s)\t(exp:%f)" or "%d\t%f" + aux.say(fmt, i, fill_elapsed, fill_rate, exp_elapsed) +end diff --git a/src/ext/timeouts/bench/bench-aux.lua b/src/ext/timeouts/bench/bench-aux.lua new file mode 100644 index 0000000000..6321247421 --- /dev/null +++ b/src/ext/timeouts/bench/bench-aux.lua @@ -0,0 +1,30 @@ +local bench = require"bench" +local clock = bench.clock + +local aux = {} + +local function time_return(begun, ...) + local duration = clock() - begun + return duration, ... +end + +function aux.time(f, ...) + local begun = clock() + return time_return(begun, f(...)) +end + +function aux.say(...) + print(string.format(...)) +end + +function aux.toboolean(s) + return tostring(s):match("^[1TtYy]") and true or false +end + +function aux.optenv(k, def) + local s = os.getenv(k) + + return (s and #s > 0 and s) or def +end + +return aux diff --git a/src/ext/timeouts/bench/bench-del.lua b/src/ext/timeouts/bench/bench-del.lua new file mode 100755 index 0000000000..4306745f21 --- /dev/null +++ b/src/ext/timeouts/bench/bench-del.lua @@ -0,0 +1,25 @@ +#!/usr/bin/env lua + +local bench = require"bench" +local aux = require"bench-aux" + +local lib = ... or aux.optenv("BENCH_L", "bench-wheel.so") +local limit = tonumber(aux.optenv("BENCH_N", 1000000)) +local step = tonumber(aux.optenv("BENCH_S", limit / 100)) +local verbose = aux.toboolean(os.getenv("BENCH_V", false)) + +local B = bench.new(lib, count) + +for i=0,limit,step do + -- add i timeouts + local fill_elapsed, fill_count = aux.time(B.fill, B, i, 60 * 1000000) + assert(i == fill_count) + + --- delete i timeouts + local del_elapsed = aux.time(B.del, B, 0, fill_count) + assert(B:empty()) + local del_rate = i > 0 and i / del_elapsed or 0 + + local fmt = verbose and "%d\t%f\t(%d/s)\t(fill:%f)" or "%d\t%f" + aux.say(fmt, i, del_elapsed, del_rate, fill_elapsed) +end diff --git a/src/ext/timeouts/bench/bench-expire.lua b/src/ext/timeouts/bench/bench-expire.lua new file mode 100755 index 0000000000..3e6374ed52 --- /dev/null +++ b/src/ext/timeouts/bench/bench-expire.lua @@ -0,0 +1,29 @@ +#!/usr/bin/env lua + +local bench = require"bench" +local aux = require"bench-aux" + +local lib = ... or aux.optenv("BENCH_L", "bench-wheel.so") +local limit = tonumber(aux.optenv("BENCH_N", 1000000)) +local step = tonumber(aux.optenv("BENCH_S", limit / 100)) +-- expire 1/1000 * #timeouts per clock update +local exp_step = tonumber(aux.optenv("BENCH_E", 0.0001)) +local verbose = aux.toboolean(os.getenv("BENCH_V", false)) + +local B = require"bench".new(lib, count) + +for i=0,limit,step do + -- add i timeouts + local fill_elapsed, fill_count, fill_last = aux.time(B.fill, B, i) + + -- expire timeouts by iteratively updating clock. exp_step is the + -- approximate number of timeouts (as a fraction of the total number + -- of timeouts) that will expire per update. + local exp_elapsed, exp_count = aux.time(B.expire, B, fill_count, math.floor(fill_last * exp_step)) + assert(exp_count == i) + assert(B:empty()) + local exp_rate = i > 0 and i / exp_elapsed or 0 + + local fmt = verbose and "%d\t%f\t(%d/s)\t(fill:%f)" or "%d\t%f" + aux.say(fmt, i, exp_elapsed, exp_rate, fill_elapsed) +end diff --git a/src/ext/timeouts/bench/bench-heap.c b/src/ext/timeouts/bench/bench-heap.c new file mode 100644 index 0000000000..f1166a4d7e --- /dev/null +++ b/src/ext/timeouts/bench/bench-heap.c @@ -0,0 +1,236 @@ +/* + * Copyright (c) 2006 Maxim Yegorushkin <maxim.yegorushkin@gmail.com> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _MIN_HEAP_H_ +#define _MIN_HEAP_H_ + +#include <stdlib.h> +#include <err.h> +#include "timeout.h" +#include "bench.h" + +#define min_heap_idx interval + +typedef timeout_t min_heap_idx_t; + +typedef struct min_heap +{ + struct timeout** p; + unsigned n, a; + timeout_t curtime; +} min_heap_t; + +static inline void min_heap_ctor(min_heap_t* s); +static inline void min_heap_dtor(min_heap_t* s); +static inline void min_heap_elem_init(struct timeout* e); +static inline int min_heap_elem_greater(struct timeout *a, struct timeout *b); +static inline int min_heap_empty(min_heap_t* s); +static inline unsigned min_heap_size(min_heap_t* s); +static inline struct timeout* min_heap_top(min_heap_t* s); +static inline int min_heap_reserve(min_heap_t* s, unsigned n); +static inline int min_heap_push(min_heap_t* s, struct timeout* e); +static inline struct timeout* min_heap_pop(min_heap_t* s); +static inline int min_heap_erase(min_heap_t* s, struct timeout* e); +static inline void min_heap_shift_up_(min_heap_t* s, unsigned hole_index, struct timeout* e); +static inline void min_heap_shift_down_(min_heap_t* s, unsigned hole_index, struct timeout* e); + +int min_heap_elem_greater(struct timeout *a, struct timeout *b) +{ + return a->expires > b->expires; +} + +void min_heap_ctor(min_heap_t* s) { s->p = 0; s->n = 0; s->a = 0; } +void min_heap_dtor(min_heap_t* s) { if(s->p) free(s->p); } +void min_heap_elem_init(struct timeout* e) { e->min_heap_idx = -1; } +int min_heap_empty(min_heap_t* s) { return 0u == s->n; } +unsigned min_heap_size(min_heap_t* s) { return s->n; } +struct timeout* min_heap_top(min_heap_t* s) { return s->n ? *s->p : 0; } + +int min_heap_push(min_heap_t* s, struct timeout* e) +{ + if(min_heap_reserve(s, s->n + 1)) + return -1; + min_heap_shift_up_(s, s->n++, e); + return 0; +} + +struct timeout* min_heap_pop(min_heap_t* s) +{ + if(s->n) + { + struct timeout* e = *s->p; + min_heap_shift_down_(s, 0u, s->p[--s->n]); + e->min_heap_idx = -1; + return e; + } + return 0; +} + +int min_heap_erase(min_heap_t* s, struct timeout* e) +{ + if(((min_heap_idx_t)-1) != e->min_heap_idx) + { + struct timeout *last = s->p[--s->n]; + unsigned parent = (e->min_heap_idx - 1) / 2; + /* we replace e with the last element in the heap. We might need to + shift it upward if it is less than its parent, or downward if it is + greater than one or both its children. Since the children are known + to be less than the parent, it can't need to shift both up and + down. */ + if (e->min_heap_idx > 0 && min_heap_elem_greater(s->p[parent], last)) + min_heap_shift_up_(s, e->min_heap_idx, last); + else + min_heap_shift_down_(s, e->min_heap_idx, last); + e->min_heap_idx = -1; + return 0; + } + return -1; +} + +int min_heap_reserve(min_heap_t* s, unsigned n) +{ + if(s->a < n) + { + struct timeout** p; + unsigned a = s->a ? s->a * 2 : 8; + if(a < n) + a = n; + if(!(p = (struct timeout**)realloc(s->p, a * sizeof *p))) + return -1; + s->p = p; + s->a = a; + } + return 0; +} + +void min_heap_shift_up_(min_heap_t* s, unsigned hole_index, struct timeout* e) +{ + unsigned parent = (hole_index - 1) / 2; + while(hole_index && min_heap_elem_greater(s->p[parent], e)) + { + (s->p[hole_index] = s->p[parent])->min_heap_idx = hole_index; + hole_index = parent; + parent = (hole_index - 1) / 2; + } + (s->p[hole_index] = e)->min_heap_idx = hole_index; +} + +void min_heap_shift_down_(min_heap_t* s, unsigned hole_index, struct timeout* e) +{ + unsigned min_child = 2 * (hole_index + 1); + while(min_child <= s->n) + { + min_child -= min_child == s->n || min_heap_elem_greater(s->p[min_child], s->p[min_child - 1]); + if(!(min_heap_elem_greater(e, s->p[min_child]))) + break; + (s->p[hole_index] = s->p[min_child])->min_heap_idx = hole_index; + hole_index = min_child; + min_child = 2 * (hole_index + 1); + } + min_heap_shift_up_(s, hole_index, e); +} + +#endif /* _MIN_HEAP_H_ */ + + +static void *init(struct timeout *timeout, size_t count, int verbose) { + min_heap_t *H; + size_t i; + + H = calloc(1, sizeof *H); + + min_heap_ctor(H); + if (0 != min_heap_reserve(H, count)) + err(1, "realloc"); + + for (i = 0; i < count; i++) { + min_heap_elem_init(&timeout[i]); + } + + return H; +} /* init() */ + + +static void add(void *ctx, struct timeout *to, timeout_t expires) { + min_heap_t *H = ctx; + min_heap_erase(H, to); + to->expires = H->curtime + expires; + if (0 != min_heap_push(H, to)) + err(1, "realloc"); +} /* add() */ + + +static void del(void *ctx, struct timeout *to) { + min_heap_erase(ctx, to); +} /* del() */ + + +static struct timeout *get(void *ctx) { + min_heap_t *H = ctx; + struct timeout *to; + + if ((to = min_heap_top(H)) && to->expires <= H->curtime) + return min_heap_pop(H); + + return NULL; +} /* get() */ + + +static void update(void *ctx, timeout_t ts) { + min_heap_t *H = ctx; + H->curtime = ts; +} /* update() */ + + +static void check(void *ctx) { + return; +} /* check() */ + + +static int empty(void *ctx) { + min_heap_t *H = ctx; + + return (NULL == min_heap_top(H)); +} /* empty() */ + + +static void destroy(void *H) { + free(H); + return; +} /* destroy() */ + + +const struct benchops benchops = { + .init = &init, + .add = &add, + .del = &del, + .get = &get, + .update = &update, + .check = &check, + .empty = &empty, + .destroy = &destroy, +}; + diff --git a/src/ext/timeouts/bench/bench-llrb.c b/src/ext/timeouts/bench/bench-llrb.c new file mode 100644 index 0000000000..bdb02f0704 --- /dev/null +++ b/src/ext/timeouts/bench/bench-llrb.c @@ -0,0 +1,425 @@ +/* ========================================================================== + * llrb.h - Iterative Left-leaning Red-Black Tree. + * -------------------------------------------------------------------------- + * Copyright (c) 2011, 2013 William Ahern <william@25thandClement.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the + * following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * -------------------------------------------------------------------------- + * CREDITS: + * o Algorithm courtesy of Robert Sedgewick, "Left-leaning Red-Black + * Trees" (September 2008); and Robert Sedgewick and Kevin Wayne, + * Algorithms (4th ed. 2011). + * + * Sedgewick touts the simplicity of the recursive implementation, + * but at least for the 2-3 tree variant the iterative approach is + * almost line-for-line identical. The magic of C pointers helps; + * it'd be uglier with Java. + * + * A couple of missing NULL checks were added to Sedgewick's deletion + * example, and insert was optimized to short-circuit rotations when + * walking up the tree. + * + * o Code implemented in the fashion of Niels Provos' excellent *BSD + * sys/tree.h pre-processor library. + * + * Regarding relative performance, I've refrained from sharing my own + * benchmarks. Differences in run-time speed were too correlated to + * compiler options and other external factors. + * + * Provos' delete implementation doesn't need to start at the root of + * the tree. However, RB_REMOVE must be passed the actual node to be + * removed. LLRB_REMOVE merely requires a key, much like + * RB_FIND/LLRB_FIND. + * ========================================================================== + */ +#ifndef LLRB_H +#define LLRB_H + +#define LLRB_VENDOR "william@25thandClement.com" +#define LLRB_VERSION 0x20130925 + +#ifndef LLRB_STATIC +#ifdef __GNUC__ +#define LLRB_STATIC __attribute__((__unused__)) static +#else +#define LLRB_STATIC static +#endif +#endif + +#define LLRB_HEAD(name, type) \ +struct name { struct type *rbh_root; } + +#define LLRB_INITIALIZER(root) { 0 } + +#define LLRB_INIT(root) do { (root)->rbh_root = 0; } while (0) + +#define LLRB_BLACK 0 +#define LLRB_RED 1 + +#define LLRB_ENTRY(type) \ +struct { struct type *rbe_left, *rbe_right, *rbe_parent; _Bool rbe_color; } + +#define LLRB_LEFT(elm, field) (elm)->field.rbe_left +#define LLRB_RIGHT(elm, field) (elm)->field.rbe_right +#define LLRB_PARENT(elm, field) (elm)->field.rbe_parent +#define LLRB_EDGE(head, elm, field) (((elm) == LLRB_ROOT(head))? &LLRB_ROOT(head) : ((elm) == LLRB_LEFT(LLRB_PARENT((elm), field), field))? &LLRB_LEFT(LLRB_PARENT((elm), field), field) : &LLRB_RIGHT(LLRB_PARENT((elm), field), field)) +#define LLRB_COLOR(elm, field) (elm)->field.rbe_color +#define LLRB_ROOT(head) (head)->rbh_root +#define LLRB_EMPTY(head) ((head)->rbh_root == 0) +#define LLRB_ISRED(elm, field) ((elm) && LLRB_COLOR((elm), field) == LLRB_RED) + +#define LLRB_PROTOTYPE(name, type, field, cmp) \ + LLRB_PROTOTYPE_INTERNAL(name, type, field, cmp,) +#define LLRB_PROTOTYPE_STATIC(name, type, field, cmp) \ + LLRB_PROTOTYPE_INTERNAL(name, type, field, cmp, LLRB_STATIC) +#define LLRB_PROTOTYPE_INTERNAL(name, type, field, cmp, attr) \ +attr struct type *name##_LLRB_INSERT(struct name *, struct type *); \ +attr struct type *name##_LLRB_DELETE(struct name *, struct type *); \ +attr struct type *name##_LLRB_FIND(struct name *, struct type *); \ +attr struct type *name##_LLRB_MIN(struct type *); \ +attr struct type *name##_LLRB_MAX(struct type *); \ +attr struct type *name##_LLRB_NEXT(struct type *); + +#define LLRB_GENERATE(name, type, field, cmp) \ + LLRB_GENERATE_INTERNAL(name, type, field, cmp,) +#define LLRB_GENERATE_STATIC(name, type, field, cmp) \ + LLRB_GENERATE_INTERNAL(name, type, field, cmp, LLRB_STATIC) +#define LLRB_GENERATE_INTERNAL(name, type, field, cmp, attr) \ +static inline void name##_LLRB_ROTL(struct type **pivot) { \ + struct type *a = *pivot; \ + struct type *b = LLRB_RIGHT(a, field); \ + if ((LLRB_RIGHT(a, field) = LLRB_LEFT(b, field))) \ + LLRB_PARENT(LLRB_RIGHT(a, field), field) = a; \ + LLRB_LEFT(b, field) = a; \ + LLRB_COLOR(b, field) = LLRB_COLOR(a, field); \ + LLRB_COLOR(a, field) = LLRB_RED; \ + LLRB_PARENT(b, field) = LLRB_PARENT(a, field); \ + LLRB_PARENT(a, field) = b; \ + *pivot = b; \ +} \ +static inline void name##_LLRB_ROTR(struct type **pivot) { \ + struct type *b = *pivot; \ + struct type *a = LLRB_LEFT(b, field); \ + if ((LLRB_LEFT(b, field) = LLRB_RIGHT(a, field))) \ + LLRB_PARENT(LLRB_LEFT(b, field), field) = b; \ + LLRB_RIGHT(a, field) = b; \ + LLRB_COLOR(a, field) = LLRB_COLOR(b, field); \ + LLRB_COLOR(b, field) = LLRB_RED; \ + LLRB_PARENT(a, field) = LLRB_PARENT(b, field); \ + LLRB_PARENT(b, field) = a; \ + *pivot = a; \ +} \ +static inline void name##_LLRB_FLIP(struct type *root) { \ + LLRB_COLOR(root, field) = !LLRB_COLOR(root, field); \ + LLRB_COLOR(LLRB_LEFT(root, field), field) = !LLRB_COLOR(LLRB_LEFT(root, field), field); \ + LLRB_COLOR(LLRB_RIGHT(root, field), field) = !LLRB_COLOR(LLRB_RIGHT(root, field), field); \ +} \ +static inline void name##_LLRB_FIXUP(struct type **root) { \ + if (LLRB_ISRED(LLRB_RIGHT(*root, field), field) && !LLRB_ISRED(LLRB_LEFT(*root, field), field)) \ + name##_LLRB_ROTL(root); \ + if (LLRB_ISRED(LLRB_LEFT(*root, field), field) && LLRB_ISRED(LLRB_LEFT(LLRB_LEFT(*root, field), field), field)) \ + name##_LLRB_ROTR(root); \ + if (LLRB_ISRED(LLRB_LEFT(*root, field), field) && LLRB_ISRED(LLRB_RIGHT(*root, field), field)) \ + name##_LLRB_FLIP(*root); \ +} \ +attr struct type *name##_LLRB_INSERT(struct name *head, struct type *elm) { \ + struct type **root = &LLRB_ROOT(head); \ + struct type *parent = 0; \ + while (*root) { \ + int comp = (cmp)((elm), (*root)); \ + parent = *root; \ + if (comp < 0) \ + root = &LLRB_LEFT(*root, field); \ + else if (comp > 0) \ + root = &LLRB_RIGHT(*root, field); \ + else \ + return *root; \ + } \ + LLRB_LEFT((elm), field) = 0; \ + LLRB_RIGHT((elm), field) = 0; \ + LLRB_COLOR((elm), field) = LLRB_RED; \ + LLRB_PARENT((elm), field) = parent; \ + *root = (elm); \ + while (parent && (LLRB_ISRED(LLRB_LEFT(parent, field), field) || LLRB_ISRED(LLRB_RIGHT(parent, field), field))) { \ + root = LLRB_EDGE(head, parent, field); \ + parent = LLRB_PARENT(parent, field); \ + name##_LLRB_FIXUP(root); \ + } \ + LLRB_COLOR(LLRB_ROOT(head), field) = LLRB_BLACK; \ + return 0; \ +} \ +static inline void name##_LLRB_MOVL(struct type **pivot) { \ + name##_LLRB_FLIP(*pivot); \ + if (LLRB_ISRED(LLRB_LEFT(LLRB_RIGHT(*pivot, field), field), field)) { \ + name##_LLRB_ROTR(&LLRB_RIGHT(*pivot, field)); \ + name##_LLRB_ROTL(pivot); \ + name##_LLRB_FLIP(*pivot); \ + } \ +} \ +static inline void name##_LLRB_MOVR(struct type **pivot) { \ + name##_LLRB_FLIP(*pivot); \ + if (LLRB_ISRED(LLRB_LEFT(LLRB_LEFT(*pivot, field), field), field)) { \ + name##_LLRB_ROTR(pivot); \ + name##_LLRB_FLIP(*pivot); \ + } \ +} \ +static inline struct type *name##_DELETEMIN(struct name *head, struct type **root) { \ + struct type **pivot = root, *deleted, *parent; \ + while (LLRB_LEFT(*pivot, field)) { \ + if (!LLRB_ISRED(LLRB_LEFT(*pivot, field), field) && !LLRB_ISRED(LLRB_LEFT(LLRB_LEFT(*pivot, field), field), field)) \ + name##_LLRB_MOVL(pivot); \ + pivot = &LLRB_LEFT(*pivot, field); \ + } \ + deleted = *pivot; \ + parent = LLRB_PARENT(*pivot, field); \ + *pivot = 0; \ + while (root != pivot) { \ + pivot = LLRB_EDGE(head, parent, field); \ + parent = LLRB_PARENT(parent, field); \ + name##_LLRB_FIXUP(pivot); \ + } \ + return deleted; \ +} \ +attr struct type *name##_LLRB_DELETE(struct name *head, struct type *elm) { \ + struct type **root = &LLRB_ROOT(head), *parent = 0, *deleted = 0; \ + int comp; \ + while (*root) { \ + parent = LLRB_PARENT(*root, field); \ + comp = (cmp)(elm, *root); \ + if (comp < 0) { \ + if (LLRB_LEFT(*root, field) && !LLRB_ISRED(LLRB_LEFT(*root, field), field) && !LLRB_ISRED(LLRB_LEFT(LLRB_LEFT(*root, field), field), field)) \ + name##_LLRB_MOVL(root); \ + root = &LLRB_LEFT(*root, field); \ + } else { \ + if (LLRB_ISRED(LLRB_LEFT(*root, field), field)) { \ + name##_LLRB_ROTR(root); \ + comp = (cmp)(elm, *root); \ + } \ + if (!comp && !LLRB_RIGHT(*root, field)) { \ + deleted = *root; \ + *root = 0; \ + break; \ + } \ + if (LLRB_RIGHT(*root, field) && !LLRB_ISRED(LLRB_RIGHT(*root, field), field) && !LLRB_ISRED(LLRB_LEFT(LLRB_RIGHT(*root, field), field), field)) { \ + name##_LLRB_MOVR(root); \ + comp = (cmp)(elm, *root); \ + } \ + if (!comp) { \ + struct type *orphan = name##_DELETEMIN(head, &LLRB_RIGHT(*root, field)); \ + LLRB_COLOR(orphan, field) = LLRB_COLOR(*root, field); \ + LLRB_PARENT(orphan, field) = LLRB_PARENT(*root, field); \ + if ((LLRB_RIGHT(orphan, field) = LLRB_RIGHT(*root, field))) \ + LLRB_PARENT(LLRB_RIGHT(orphan, field), field) = orphan; \ + if ((LLRB_LEFT(orphan, field) = LLRB_LEFT(*root, field))) \ + LLRB_PARENT(LLRB_LEFT(orphan, field), field) = orphan; \ + deleted = *root; \ + *root = orphan; \ + parent = *root; \ + break; \ + } else \ + root = &LLRB_RIGHT(*root, field); \ + } \ + } \ + while (parent) { \ + root = LLRB_EDGE(head, parent, field); \ + parent = LLRB_PARENT(parent, field); \ + name##_LLRB_FIXUP(root); \ + } \ + if (LLRB_ROOT(head)) \ + LLRB_COLOR(LLRB_ROOT(head), field) = LLRB_BLACK; \ + return deleted; \ +} \ +attr struct type *name##_LLRB_FIND(struct name *head, struct type *key) { \ + struct type *elm = LLRB_ROOT(head); \ + while (elm) { \ + int comp = (cmp)(key, elm); \ + if (comp < 0) \ + elm = LLRB_LEFT(elm, field); \ + else if (comp > 0) \ + elm = LLRB_RIGHT(elm, field); \ + else \ + return elm; \ + } \ + return 0; \ +} \ +attr struct type *name##_LLRB_MIN(struct type *elm) { \ + while (elm && LLRB_LEFT(elm, field)) \ + elm = LLRB_LEFT(elm, field); \ + return elm; \ +} \ +attr struct type *name##_LLRB_MAX(struct type *elm) { \ + while (elm && LLRB_RIGHT(elm, field)) \ + elm = LLRB_RIGHT(elm, field); \ + return elm; \ +} \ +attr struct type *name##_LLRB_NEXT(struct type *elm) { \ + if (LLRB_RIGHT(elm, field)) { \ + return name##_LLRB_MIN(LLRB_RIGHT(elm, field)); \ + } else if (LLRB_PARENT(elm, field)) { \ + if (elm == LLRB_LEFT(LLRB_PARENT(elm, field), field)) \ + return LLRB_PARENT(elm, field); \ + while (LLRB_PARENT(elm, field) && elm == LLRB_RIGHT(LLRB_PARENT(elm, field), field)) \ + elm = LLRB_PARENT(elm, field); \ + return LLRB_PARENT(elm, field); \ + } else return 0; \ +} + +#define LLRB_INSERT(name, head, elm) name##_LLRB_INSERT((head), (elm)) +#define LLRB_DELETE(name, head, elm) name##_LLRB_DELETE((head), (elm)) +#define LLRB_REMOVE(name, head, elm) name##_LLRB_DELETE((head), (elm)) +#define LLRB_FIND(name, head, elm) name##_LLRB_FIND((head), (elm)) +#define LLRB_MIN(name, head) name##_LLRB_MIN(LLRB_ROOT((head))) +#define LLRB_MAX(name, head) name##_LLRB_MAX(LLRB_ROOT((head))) +#define LLRB_NEXT(name, head, elm) name##_LLRB_NEXT((elm)) + +#define LLRB_FOREACH(elm, name, head) \ +for ((elm) = LLRB_MIN(name, head); (elm); (elm) = name##_LLRB_NEXT((elm))) + +#endif /* LLRB_H */ + + +#include <stdlib.h> + +#include "timeout.h" +#include "bench.h" + + +struct rbtimeout { + timeout_t expires; + + int pending; + + LLRB_ENTRY(rbtimeout) rbe; +}; + +struct rbtimeouts { + timeout_t curtime; + LLRB_HEAD(tree, rbtimeout) tree; +}; + + +static int timeoutcmp(struct rbtimeout *a, struct rbtimeout *b) { + if (a->expires < b->expires) { + return -1; + } else if (a->expires > b->expires) { + return 1; + } else if (a < b) { + return -1; + } else if (a > b) { + return 1; + } else { + return 0; + } +} /* timeoutcmp() */ + +LLRB_GENERATE_STATIC(tree, rbtimeout, rbe, timeoutcmp) + +static void *init(struct timeout *timeout, size_t count, int verbose) { + struct rbtimeouts *T; + size_t i; + + T = malloc(sizeof *T); + T->curtime = 0; + LLRB_INIT(&T->tree); + + for (i = 0; i < count; i++) { + struct rbtimeout *to = (void *)&timeout[i]; + to->expires = 0; + to->pending = 0; + } + + return T; +} /* init() */ + + +static void add(void *ctx, struct timeout *_to, timeout_t expires) { + struct rbtimeouts *T = ctx; + struct rbtimeout *to = (void *)_to; + + if (to->pending) + LLRB_REMOVE(tree, &T->tree, to); + + to->expires = T->curtime + expires; + LLRB_INSERT(tree, &T->tree, to); + to->pending = 1; +} /* add() */ + + +static void del(void *ctx, struct timeout *_to) { + struct rbtimeouts *T = ctx; + struct rbtimeout *to = (void *)_to; + + LLRB_REMOVE(tree, &T->tree, to); + to->pending = 0; + to->expires = 0; +} /* del() */ + + +static struct timeout *get(void *ctx) { + struct rbtimeouts *T = ctx; + struct rbtimeout *to; + + if ((to = LLRB_MIN(tree, &T->tree)) && to->expires <= T->curtime) { + LLRB_REMOVE(tree, &T->tree, to); + to->pending = 0; + to->expires = 0; + + return (void *)to; + } + + return NULL; +} /* get() */ + + +static void update(void *ctx, timeout_t ts) { + struct rbtimeouts *T = ctx; + T->curtime = ts; +} /* update() */ + + +static void check(void *ctx) { + return; +} /* check() */ + + +static int empty(void *ctx) { + struct rbtimeouts *T = ctx; + + return LLRB_EMPTY(&T->tree); +} /* empty() */ + + +static void destroy(void *ctx) { + free(ctx); + return; +} /* destroy() */ + + +const struct benchops benchops = { + .init = &init, + .add = &add, + .del = &del, + .get = &get, + .update = &update, + .check = &check, + .empty = &empty, + .destroy = &destroy, +}; + diff --git a/src/ext/timeouts/bench/bench-wheel.c b/src/ext/timeouts/bench/bench-wheel.c new file mode 100644 index 0000000000..0cba1af83e --- /dev/null +++ b/src/ext/timeouts/bench/bench-wheel.c @@ -0,0 +1,81 @@ +#include <stdlib.h> + +#define TIMEOUT_PUBLIC static + +#include "timeout.h" +#include "timeout.c" +#include "bench.h" + + +static void *init(struct timeout *timeout, size_t count, int verbose) { + struct timeouts *T; + size_t i; + int error; + + T = timeouts_open(TIMEOUT_mHZ, &error); + + for (i = 0; i < count; i++) { + timeout_init(&timeout[i], 0); + } + +#if TIMEOUT_DEBUG - 0 + timeout_debug = verbose; +#endif + + return T; +} /* init() */ + + +static void add(void *T, struct timeout *to, timeout_t expires) { + timeouts_add(T, to, expires); +} /* add() */ + + +static void del(void *T, struct timeout *to) { + timeouts_del(T, to); +} /* del() */ + + +static struct timeout *get(void *T) { + return timeouts_get(T); +} /* get() */ + + +static void update(void *T, timeout_t ts) { + timeouts_update(T, ts); +} /* update() */ + + +static void (check)(void *T) { + if (!timeouts_check(T, stderr)) + _Exit(1); +} /* check() */ + + +static int empty(void *T) { + return !(timeouts_pending(T) || timeouts_expired(T)); +} /* empty() */ + + +static struct timeout *next(void *T, struct timeouts_it *it) { + return timeouts_next(T, it); +} /* next() */ + + +static void destroy(void *T) { + timeouts_close(T); +} /* destroy() */ + + +const struct benchops benchops = { + .init = &init, + .add = &add, + .del = &del, + .get = &get, + .update = &update, + .check = &check, + .empty = &empty, + .next = &next, + .destroy = &destroy +}; + diff --git a/src/ext/timeouts/bench/bench.c b/src/ext/timeouts/bench/bench.c new file mode 100644 index 0000000000..0d4cee44a0 --- /dev/null +++ b/src/ext/timeouts/bench/bench.c @@ -0,0 +1,293 @@ +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <errno.h> +#include <unistd.h> +#include <dlfcn.h> + +#if __APPLE__ +#include <mach/mach_time.h> +#endif + +#include <lua.h> +#include <lualib.h> +#include <lauxlib.h> + +#include "timeout.h" +#include "bench.h" + +#if LUA_VERSION_NUM < 502 +static int lua_absindex(lua_State *L, int idx) { + return (idx > 0 || idx <= LUA_REGISTRYINDEX)? idx : lua_gettop(L) + idx + 1; +} /* lua_absindex() */ + +static void luaL_setfuncs(lua_State *L, const luaL_Reg *l, int nup) { + int i, t = lua_absindex(L, -1 - nup); + + for (; l->name; l++) { + for (i = 0; i < nup; i++) + lua_pushvalue(L, -nup); + lua_pushcclosure(L, l->func, nup); + lua_setfield(L, t, l->name); + } + + lua_pop(L, nup); +} /* luaL_setfuncs() */ + +#define luaL_newlibtable(L, l) \ + lua_createtable(L, 0, (sizeof (l) / sizeof *(l)) - 1) + +#define luaL_newlib(L, l) \ + (luaL_newlibtable((L), (l)), luaL_setfuncs((L), (l), 0)) +#endif + +#ifndef MAX +#define MAX(a, b) (((a) > (b))? (a) : (b)) +#endif + + +struct bench { + const char *path; + void *solib; + size_t count; + timeout_t timeout_max; + int verbose; + + void *state; + struct timeout *timeout; + struct benchops ops; + timeout_t curtime; +}; /* struct bench */ + + +#if __APPLE__ +static mach_timebase_info_data_t timebase; +#endif + + +static int long long monotime(void) { +#if __APPLE__ + unsigned long long abt; + + abt = mach_absolute_time(); + abt = abt * timebase.numer / timebase.denom; + + return abt / 1000LL; +#else + struct timespec ts; + + clock_gettime(CLOCK_MONOTONIC, &ts); + + return (ts.tv_sec * 1000000L) + (ts.tv_nsec / 1000L); +#endif +} /* monotime() */ + + +static int bench_clock(lua_State *L) { + lua_pushnumber(L, (double)monotime() / 1000000L); + + return 1; +} /* bench_clock() */ + + +static int bench_new(lua_State *L) { + const char *path = luaL_checkstring(L, 1); + size_t count = luaL_optinteger(L, 2, 1000000); + timeout_t timeout_max = luaL_optinteger(L, 3, 300 * 1000000L); + int verbose = (lua_isnone(L, 4))? 0 : lua_toboolean(L, 4); + struct bench *B; + struct benchops *ops; + + B = lua_newuserdata(L, sizeof *B); + memset(B, 0, sizeof *B); + + luaL_getmetatable(L, "BENCH*"); + lua_setmetatable(L, -2); + + B->count = count; + B->timeout_max = timeout_max; + B->verbose = verbose; + + if (!(B->timeout = calloc(count, sizeof *B->timeout))) + return luaL_error(L, "%s", strerror(errno)); + + if (!(B->solib = dlopen(path, RTLD_NOW|RTLD_LOCAL))) + return luaL_error(L, "%s: %s", path, dlerror()); + + if (!(ops = dlsym(B->solib, "benchops"))) + return luaL_error(L, "%s: %s", path, dlerror()); + + B->ops = *ops; + B->state = B->ops.init(B->timeout, B->count, B->verbose); + + return 1; +} /* bench_new() */ + + +static int bench_add(lua_State *L) { + struct bench *B = lua_touserdata(L, 1); + unsigned i; + timeout_t t; + + i = (lua_isnoneornil(L, 2))? random() % B->count : (unsigned)luaL_checkinteger(L, 2); + t = (lua_isnoneornil(L, 3))? random() % B->timeout_max : (unsigned)luaL_checkinteger(L, 3); + + B->ops.add(B->state, &B->timeout[i], t); + + return 0; +} /* bench_add() */ + + +static int bench_del(lua_State *L) { + struct bench *B = lua_touserdata(L, 1); + size_t i = luaL_optinteger(L, 2, random() % B->count); + size_t j = luaL_optinteger(L, 3, i); + + while (i <= j && i < B->count) { + B->ops.del(B->state, &B->timeout[i]); + ++i; + } + + return 0; +} /* bench_del() */ + + +static int bench_fill(lua_State *L) { + struct bench *B = lua_touserdata(L, 1); + size_t count = luaL_optinteger(L, 2, B->count); + long timeout_inc = luaL_optinteger(L, 3, -1), timeout_max = 0, timeout; + size_t i; + + if (timeout_inc < 0) { + for (i = 0; i < count; i++) { + timeout = random() % B->timeout_max; + B->ops.add(B->state, &B->timeout[i], timeout); + timeout_max = MAX(timeout, timeout_max); + } + } else { + for (i = 0; i < count; i++) { + timeout = timeout_inc + i; + B->ops.add(B->state, &B->timeout[i], timeout_inc + i); + timeout_max = MAX(timeout, timeout_max); + } + } + + lua_pushinteger(L, (lua_Integer)count); + lua_pushinteger(L, (lua_Integer)timeout_max); + + return 2; +} /* bench_fill() */ + + +static int bench_expire(lua_State *L) { + struct bench *B = lua_touserdata(L, 1); + unsigned count = luaL_optinteger(L, 2, B->count); + unsigned step = luaL_optinteger(L, 3, 300000); + size_t i = 0; + + while (i < count && !B->ops.empty(B->state)) { + B->curtime += step; + B->ops.update(B->state, B->curtime); + + while (B->ops.get(B->state)) + i++; + } + + lua_pushinteger(L, (lua_Integer)i); + + return 1; +} /* bench_expire() */ + + +static int bench_empty(lua_State *L) { + struct bench *B = lua_touserdata(L, 1); + + lua_pushboolean(L, B->ops.empty(B->state)); + + return 1; +} /* bench_empty() */ + + +static int bench__next(lua_State *L) { + struct bench *B = lua_touserdata(L, lua_upvalueindex(1)); + struct timeouts_it *it = lua_touserdata(L, lua_upvalueindex(2)); + struct timeout *to; + + if (!B->ops.next || !(to = B->ops.next(B->state, it))) + return 0; + + lua_pushinteger(L, luaL_optinteger(L, 2, 0) + 1); + + lua_newtable(L); + lua_pushinteger(L, to->expires); + lua_setfield(L, -2, "expires"); + + return 2; +} /* bench__next() */ + +static int bench__pairs(lua_State *L) { + struct timeouts_it *it; + + lua_settop(L, 1); + + it = lua_newuserdata(L, sizeof *it); + TIMEOUTS_IT_INIT(it, TIMEOUTS_ALL); + + lua_pushcclosure(L, &bench__next, 2); + lua_pushvalue(L, 1); + lua_pushinteger(L, 0); + + return 3; +} /* bench__pairs() */ + + +static int bench__gc(lua_State *L) { + struct bench *B = lua_touserdata(L, 1); + + if (B->state) { + B->ops.destroy(B->state); + B->state = NULL; + } + + return 0; +} /* bench__gc() */ + + +static const luaL_Reg bench_methods[] = { + { "add", &bench_add }, + { "del", &bench_del }, + { "fill", &bench_fill }, + { "expire", &bench_expire }, + { "empty", &bench_empty }, + { "close", &bench__gc }, + { NULL, NULL } +}; + +static const luaL_Reg bench_metatable[] = { + { "__pairs", &bench__pairs }, + { "__gc", &bench__gc }, + { NULL, NULL } +}; + +static const luaL_Reg bench_globals[] = { + { "new", &bench_new }, + { "clock", &bench_clock }, + { NULL, NULL } +}; + +int luaopen_bench(lua_State *L) { +#if __APPLE__ + mach_timebase_info(&timebase); +#endif + + if (luaL_newmetatable(L, "BENCH*")) { + luaL_setfuncs(L, bench_metatable, 0); + luaL_newlib(L, bench_methods); + lua_setfield(L, -2, "__index"); + } + + luaL_newlib(L, bench_globals); + + return 1; +} /* luaopen_bench() */ + diff --git a/src/ext/timeouts/bench/bench.h b/src/ext/timeouts/bench/bench.h new file mode 100644 index 0000000000..bc1f7cf177 --- /dev/null +++ b/src/ext/timeouts/bench/bench.h @@ -0,0 +1,11 @@ +struct benchops { + void *(*init)(struct timeout *, size_t, int); + void (*add)(void *, struct timeout *, timeout_t); + void (*del)(void *, struct timeout *); + struct timeout *(*get)(void *); + void (*update)(void *, timeout_t); + void (*check)(void *); + int (*empty)(void *); + struct timeout *(*next)(void *, struct timeouts_it *); + void (*destroy)(void *); +}; /* struct benchops() */ diff --git a/src/ext/timeouts/bench/bench.plt b/src/ext/timeouts/bench/bench.plt new file mode 100644 index 0000000000..6e143c65e1 --- /dev/null +++ b/src/ext/timeouts/bench/bench.plt @@ -0,0 +1,19 @@ +set terminal postscript color + +set key top left +set xlabel "Number of timeouts" +set ylabel "Time\n(microseconds)" +#set logscale x + +set title "Time spent installing timeouts" font ",20" +plot 'heap-add.dat' using 1:($2*1000000) title "min-heap" with lines ls 1 lw 3 lc "red", \ + 'wheel-add.dat' using 1:($2*1000000) title "hierarchical wheel" with lines ls 1 lw 3 lc "forest-green" + +set title "Time spent deleting timeouts" font ",20" +plot 'heap-del.dat' using 1:($2*1000000) title "min-heap" with lines ls 1 lw 3 lc "red", \ + 'wheel-del.dat' using 1:($2*1000000) title "hierarchical wheel" with lines ls 1 lw 3 lc "forest-green" + +set title "Time spent expiring timeouts\n(by iteratively updating clock ~1000 times)" font ",20" +plot 'heap-expire.dat' using 1:($2*1000000) title "min-heap" with lines ls 1 lw 3 lc "red", \ + 'wheel-expire.dat' using 1:($2*1000000) title "hierarchical wheel" with lines ls 1 lw 3 lc "forest-green" + diff --git a/src/ext/timeouts/lua/Rules.mk b/src/ext/timeouts/lua/Rules.mk new file mode 100644 index 0000000000..0f06fce30b --- /dev/null +++ b/src/ext/timeouts/lua/Rules.mk @@ -0,0 +1,20 @@ +$(LUA_APIS:%=$(top_builddir)/lua/%/timeout.so): $(top_srcdir)/lua/timeout-lua.c $(top_srcdir)/timeout.h $(top_srcdir)/timeout.c + mkdir -p $(@D) + @$(SHRC); echo_cmd $(CC) -o $@ $(top_srcdir)/lua/timeout-lua.c -I$(top_srcdir) -DWHEEL_BIT=$(WHEEL_BIT) -DWHEEL_NUM=$(WHEEL_NUM) $(LUA53_CPPFLAGS) $(ALL_CPPFLAGS) $(ALL_CFLAGS) $(ALL_SOFLAGS) $(ALL_LDFLAGS) $(ALL_LIBS) + +$(top_builddir)/lua/5.1/timeouts.so: $(top_builddir)/lua/5.1/timeout.so +$(top_builddir)/lua/5.2/timeouts.so: $(top_builddir)/lua/5.2/timeout.so +$(top_builddir)/lua/5.3/timeouts.so: $(top_builddir)/lua/5.3/timeout.so + +$(LUA_APIS:%=$(top_builddir)/lua/%/timeouts.so): + cd $(@D) && ln -fs timeout.so timeouts.so + +lua-5.1: $(top_builddir)/lua/5.1/timeout.so $(top_builddir)/lua/5.1/timeouts.so +lua-5.2: $(top_builddir)/lua/5.2/timeout.so $(top_builddir)/lua/5.2/timeouts.so +lua-5.3: $(top_builddir)/lua/5.3/timeout.so $(top_builddir)/lua/5.3/timeouts.so + +lua-clean: + $(RM) -r $(top_builddir)/lua/5.? + +clean: lua-clean + diff --git a/src/ext/timeouts/lua/timeout-lua.c b/src/ext/timeouts/lua/timeout-lua.c new file mode 100644 index 0000000000..4d4e54cba6 --- /dev/null +++ b/src/ext/timeouts/lua/timeout-lua.c @@ -0,0 +1,396 @@ +#include <assert.h> +#include <string.h> + +#include <lua.h> +#include <lualib.h> +#include <lauxlib.h> + +#if LUA_VERSION_NUM != 503 +#error only Lua 5.3 supported +#endif + +#define TIMEOUT_PUBLIC static +#include "timeout.h" +#include "timeout.c" + +#define TIMEOUT_METANAME "struct timeout" +#define TIMEOUTS_METANAME "struct timeouts*" + +static struct timeout * +to_checkudata(lua_State *L, int index) +{ + return luaL_checkudata(L, index, TIMEOUT_METANAME); +} + +static struct timeouts * +tos_checkudata(lua_State *L, int index) +{ + return *(struct timeouts **)luaL_checkudata(L, index, TIMEOUTS_METANAME); +} + +static void +tos_bind(lua_State *L, int tos_index, int to_index) +{ + lua_getuservalue(L, tos_index); + lua_pushlightuserdata(L, to_checkudata(L, to_index)); + lua_pushvalue(L, to_index); + lua_rawset(L, -3); + lua_pop(L, 1); +} + +static void +tos_unbind(lua_State *L, int tos_index, int to_index) +{ + lua_getuservalue(L, tos_index); + lua_pushlightuserdata(L, to_checkudata(L, to_index)); + lua_pushnil(L); + lua_rawset(L, -3); + lua_pop(L, 1); +} + +static int +to__index(lua_State *L) +{ + struct timeout *to = to_checkudata(L, 1); + + if (lua_type(L, 2 == LUA_TSTRING)) { + const char *key = lua_tostring(L, 2); + + if (!strcmp(key, "flags")) { + lua_pushinteger(L, to->flags); + + return 1; + } else if (!strcmp(key, "expires")) { + lua_pushinteger(L, to->expires); + + return 1; + } + } + + if (LUA_TNIL != lua_getuservalue(L, 1)) { + lua_pushvalue(L, 2); + if (LUA_TNIL != lua_rawget(L, -2)) + return 1; + } + + lua_pushvalue(L, 2); + if (LUA_TNIL != lua_rawget(L, lua_upvalueindex(1))) + return 1; + + return 0; +} + +static int +to__newindex(lua_State *L) +{ + if (LUA_TNIL == lua_getuservalue(L, 1)) { + lua_newtable(L); + lua_pushvalue(L, -1); + lua_setuservalue(L, 1); + } + + lua_pushvalue(L, 2); + lua_pushvalue(L, 3); + lua_rawset(L, -3); + + return 0; +} + +static int +to__gc(lua_State *L) +{ + struct timeout *to = to_checkudata(L, 1); + + /* + * NB: On script exit it's possible for a timeout to still be + * associated with a timeouts object, particularly when the timeouts + * object was created first. + */ + timeout_del(to); + + return 0; +} + +static int +to_new(lua_State *L) +{ + int flags = luaL_optinteger(L, 1, 0); + struct timeout *to; + + to = lua_newuserdata(L, sizeof *to); + timeout_init(to, flags); + luaL_setmetatable(L, TIMEOUT_METANAME); + + return 1; +} + +static const luaL_Reg to_methods[] = { + { NULL, NULL }, +}; + +static const luaL_Reg to_metatable[] = { + { "__index", &to__index }, + { "__newindex", &to__newindex }, + { "__gc", &to__gc }, + { NULL, NULL }, +}; + +static const luaL_Reg to_globals[] = { + { "new", &to_new }, + { NULL, NULL }, +}; + +static void +to_newmetatable(lua_State *L) +{ + if (luaL_newmetatable(L, TIMEOUT_METANAME)) { + /* + * fill metamethod table, capturing the methods table as an + * upvalue for use by __index metamethod + */ + luaL_newlib(L, to_methods); + luaL_setfuncs(L, to_metatable, 1); + } +} + +int +luaopen_timeout(lua_State *L) +{ + to_newmetatable(L); + + luaL_newlib(L, to_globals); + lua_pushinteger(L, TIMEOUT_INT); + lua_setfield(L, -2, "INT"); + lua_pushinteger(L, TIMEOUT_ABS); + lua_setfield(L, -2, "ABS"); + + return 1; +} + +static int +tos_update(lua_State *L) +{ + struct timeouts *T = tos_checkudata(L, 1); + lua_Number n = luaL_checknumber(L, 2); + + timeouts_update(T, timeouts_f2i(T, n)); + + lua_pushvalue(L, 1); + + return 1; +} + +static int +tos_step(lua_State *L) +{ + struct timeouts *T = tos_checkudata(L, 1); + lua_Number n = luaL_checknumber(L, 2); + + timeouts_step(T, timeouts_f2i(T, n)); + + lua_pushvalue(L, 1); + + return 1; +} + +static int +tos_timeout(lua_State *L) +{ + struct timeouts *T = tos_checkudata(L, 1); + + lua_pushnumber(L, timeouts_i2f(T, timeouts_timeout(T))); + + return 1; +} + +static int +tos_add(lua_State *L) +{ + struct timeouts *T = tos_checkudata(L, 1); + struct timeout *to = to_checkudata(L, 2); + lua_Number timeout = luaL_checknumber(L, 3); + + tos_bind(L, 1, 2); + timeouts_addf(T, to, timeout); + + return lua_pushvalue(L, 1), 1; +} + +static int +tos_del(lua_State *L) +{ + struct timeouts *T = tos_checkudata(L, 1); + struct timeout *to = to_checkudata(L, 2); + + timeouts_del(T, to); + tos_unbind(L, 1, 2); + + return lua_pushvalue(L, 1), 1; +} + +static int +tos_get(lua_State *L) +{ + struct timeouts *T = tos_checkudata(L, 1); + struct timeout *to; + + if (!(to = timeouts_get(T))) + return 0; + + lua_getuservalue(L, 1); + lua_rawgetp(L, -1, to); + + if (!timeout_pending(to)) + tos_unbind(L, 1, lua_absindex(L, -1)); + + return 1; +} + +static int +tos_pending(lua_State *L) +{ + struct timeouts *T = tos_checkudata(L, 1); + + lua_pushboolean(L, timeouts_pending(T)); + + return 1; +} + +static int +tos_expired(lua_State *L) +{ + struct timeouts *T = tos_checkudata(L, 1); + + lua_pushboolean(L, timeouts_expired(T)); + + return 1; +} + +static int +tos_check(lua_State *L) +{ + struct timeouts *T = tos_checkudata(L, 1); + + lua_pushboolean(L, timeouts_check(T, NULL)); + + return 1; +} + +static int +tos__next(lua_State *L) +{ + struct timeouts *T = tos_checkudata(L, lua_upvalueindex(1)); + struct timeouts_it *it = lua_touserdata(L, lua_upvalueindex(2)); + struct timeout *to; + + if (!(to = timeouts_next(T, it))) + return 0; + + lua_getuservalue(L, lua_upvalueindex(1)); + lua_rawgetp(L, -1, to); + + return 1; +} + +static int +tos_timeouts(lua_State *L) +{ + int flags = luaL_checkinteger(L, 2); + struct timeouts_it *it; + + tos_checkudata(L, 1); + lua_pushvalue(L, 1); + it = lua_newuserdata(L, sizeof *it); + TIMEOUTS_IT_INIT(it, flags); + lua_pushcclosure(L, &tos__next, 2); + + return 1; +} + +static int +tos__gc(lua_State *L) +{ + struct timeouts **tos = luaL_checkudata(L, 1, TIMEOUTS_METANAME); + struct timeout *to; + + TIMEOUTS_FOREACH(to, *tos, TIMEOUTS_ALL) { + timeouts_del(*tos, to); + } + + timeouts_close(*tos); + *tos = NULL; + + return 0; +} + +static int +tos_new(lua_State *L) +{ + timeout_t hz = luaL_optinteger(L, 1, 0); + struct timeouts **T; + int error; + + T = lua_newuserdata(L, sizeof *T); + luaL_setmetatable(L, TIMEOUTS_METANAME); + + lua_newtable(L); + lua_setuservalue(L, -2); + + if (!(*T = timeouts_open(hz, &error))) + return luaL_error(L, "%s", strerror(error)); + + return 1; +} + +static const luaL_Reg tos_methods[] = { + { "update", &tos_update }, + { "step", &tos_step }, + { "timeout", &tos_timeout }, + { "add", &tos_add }, + { "del", &tos_del }, + { "get", &tos_get }, + { "pending", &tos_pending }, + { "expired", &tos_expired }, + { "check", &tos_check }, + { "timeouts", &tos_timeouts }, + { NULL, NULL }, +}; + +static const luaL_Reg tos_metatable[] = { + { "__gc", &tos__gc }, + { NULL, NULL }, +}; + +static const luaL_Reg tos_globals[] = { + { "new", &tos_new }, + { NULL, NULL }, +}; + +static void +tos_newmetatable(lua_State *L) +{ + if (luaL_newmetatable(L, TIMEOUTS_METANAME)) { + luaL_setfuncs(L, tos_metatable, 0); + luaL_newlib(L, tos_methods); + lua_setfield(L, -2, "__index"); + } +} + +int +luaopen_timeouts(lua_State *L) +{ + to_newmetatable(L); + tos_newmetatable(L); + + luaL_newlib(L, tos_globals); + lua_pushinteger(L, TIMEOUTS_PENDING); + lua_setfield(L, -2, "PENDING"); + lua_pushinteger(L, TIMEOUTS_EXPIRED); + lua_setfield(L, -2, "EXPIRED"); + lua_pushinteger(L, TIMEOUTS_ALL); + lua_setfield(L, -2, "ALL"); + lua_pushinteger(L, TIMEOUTS_CLEAR); + lua_setfield(L, -2, "CLEAR"); + + return 1; +} diff --git a/src/ext/timeouts/test-timeout.c b/src/ext/timeouts/test-timeout.c new file mode 100644 index 0000000000..8077129376 --- /dev/null +++ b/src/ext/timeouts/test-timeout.c @@ -0,0 +1,530 @@ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <limits.h> + +#include "timeout.h" + +#define THE_END_OF_TIME ((timeout_t)-1) + +static int check_misc(void) { + if (TIMEOUT_VERSION != timeout_version()) + return 1; + if (TIMEOUT_V_REL != timeout_v_rel()) + return 1; + if (TIMEOUT_V_API != timeout_v_api()) + return 1; + if (TIMEOUT_V_ABI != timeout_v_abi()) + return 1; + if (strcmp(timeout_vendor(), TIMEOUT_VENDOR)) + return 1; + return 0; +} + +static int check_open_close(timeout_t hz_set, timeout_t hz_expect) { + int err=0; + struct timeouts *tos = timeouts_open(hz_set, &err); + if (!tos) + return 1; + if (err) + return 1; + if (hz_expect != timeouts_hz(tos)) + return 1; + timeouts_close(tos); + return 0; +} + +/* Not very random */ +static timeout_t random_to(timeout_t min, timeout_t max) +{ + if (max <= min) + return min; + /* Not actually all that random, but should exercise the code. */ + timeout_t rand64 = random() * (timeout_t)INT_MAX + random(); + return min + (rand64 % (max-min)); +} + +/* configuration for check_randomized */ +struct rand_cfg { + /* When creating timeouts, smallest possible delay */ + timeout_t min_timeout; + /* When creating timeouts, largest possible delay */ + timeout_t max_timeout; + /* First time to start the clock at. */ + timeout_t start_at; + /* Do not advance the clock past this time. */ + timeout_t end_at; + /* Number of timeouts to create and monitor. */ + int n_timeouts; + /* Advance the clock by no more than this each step. */ + timeout_t max_step; + /* Use relative timers and stepping */ + int relative; + /* Every time the clock ticks, try removing this many timeouts at + * random. */ + int try_removing; + /* When we're done, advance the clock to the end of time. */ + int finalize; +}; + +static int check_randomized(const struct rand_cfg *cfg) +{ +#define FAIL() do { \ + printf("Failure on line %d\n", __LINE__); \ + goto done; \ + } while (0) + + int i, err; + int rv = 1; + struct timeout *t = calloc(cfg->n_timeouts, sizeof(struct timeout)); + timeout_t *timeouts = calloc(cfg->n_timeouts, sizeof(timeout_t)); + uint8_t *fired = calloc(cfg->n_timeouts, sizeof(uint8_t)); + uint8_t *found = calloc(cfg->n_timeouts, sizeof(uint8_t)); + uint8_t *deleted = calloc(cfg->n_timeouts, sizeof(uint8_t)); + struct timeouts *tos = timeouts_open(0, &err); + timeout_t now = cfg->start_at; + int n_added_pending = 0, cnt_added_pending = 0; + int n_added_expired = 0, cnt_added_expired = 0; + struct timeouts_it it_p, it_e, it_all; + int p_done = 0, e_done = 0, all_done = 0; + struct timeout *to = NULL; + const int rel = cfg->relative; + + if (!t || !timeouts || !tos || !fired || !found || !deleted) + FAIL(); + timeouts_update(tos, cfg->start_at); + + for (i = 0; i < cfg->n_timeouts; ++i) { + if (&t[i] != timeout_init(&t[i], rel ? 0 : TIMEOUT_ABS)) + FAIL(); + if (timeout_pending(&t[i])) + FAIL(); + if (timeout_expired(&t[i])) + FAIL(); + + timeouts[i] = random_to(cfg->min_timeout, cfg->max_timeout); + + timeouts_add(tos, &t[i], timeouts[i] - (rel ? now : 0)); + if (timeouts[i] <= cfg->start_at) { + if (timeout_pending(&t[i])) + FAIL(); + if (! timeout_expired(&t[i])) + FAIL(); + ++n_added_expired; + } else { + if (! timeout_pending(&t[i])) + FAIL(); + if (timeout_expired(&t[i])) + FAIL(); + ++n_added_pending; + } + } + + if (!!n_added_pending != timeouts_pending(tos)) + FAIL(); + if (!!n_added_expired != timeouts_expired(tos)) + FAIL(); + + /* Test foreach, interleaving a few iterators. */ + TIMEOUTS_IT_INIT(&it_p, TIMEOUTS_PENDING); + TIMEOUTS_IT_INIT(&it_e, TIMEOUTS_EXPIRED); + TIMEOUTS_IT_INIT(&it_all, TIMEOUTS_ALL); + while (! (p_done && e_done && all_done)) { + if (!p_done) { + to = timeouts_next(tos, &it_p); + if (to) { + i = to - &t[0]; + ++found[i]; + ++cnt_added_pending; + } else { + p_done = 1; + } + } + if (!e_done) { + to = timeouts_next(tos, &it_e); + if (to) { + i = to - &t[0]; + ++found[i]; + ++cnt_added_expired; + } else { + e_done = 1; + } + } + if (!all_done) { + to = timeouts_next(tos, &it_all); + if (to) { + i = to - &t[0]; + ++found[i]; + } else { + all_done = 1; + } + } + } + + for (i = 0; i < cfg->n_timeouts; ++i) { + if (found[i] != 2) + FAIL(); + } + if (cnt_added_expired != n_added_expired) + FAIL(); + if (cnt_added_pending != n_added_pending) + FAIL(); + + while (NULL != (to = timeouts_get(tos))) { + i = to - &t[0]; + assert(&t[i] == to); + if (timeouts[i] > cfg->start_at) + FAIL(); /* shouldn't have happened yet */ + + --n_added_expired; /* drop expired timeouts. */ + ++fired[i]; + } + + if (n_added_expired != 0) + FAIL(); + + while (now < cfg->end_at) { + int n_fired_this_time = 0; + timeout_t first_at = timeouts_timeout(tos) + now; + + timeout_t oldtime = now; + timeout_t step = random_to(1, cfg->max_step); + int another; + now += step; + if (rel) + timeouts_step(tos, step); + else + timeouts_update(tos, now); + + for (i = 0; i < cfg->try_removing; ++i) { + int idx = random() % cfg->n_timeouts; + if (! fired[idx]) { + timeout_del(&t[idx]); + ++deleted[idx]; + } + } + + another = (timeouts_timeout(tos) == 0); + + while (NULL != (to = timeouts_get(tos))) { + if (! another) + FAIL(); /* Thought we saw the last one! */ + i = to - &t[0]; + assert(&t[i] == to); + if (timeouts[i] > now) + FAIL(); /* shouldn't have happened yet */ + if (timeouts[i] <= oldtime) + FAIL(); /* should have happened already */ + if (timeouts[i] < first_at) + FAIL(); /* first_at should've been earlier */ + fired[i]++; + n_fired_this_time++; + another = (timeouts_timeout(tos) == 0); + } + if (n_fired_this_time && first_at > now) + FAIL(); /* first_at should've been earlier */ + if (another) + FAIL(); /* Huh? We think there are more? */ + if (!timeouts_check(tos, stderr)) + FAIL(); + } + + for (i = 0; i < cfg->n_timeouts; ++i) { + if (fired[i] > 1) + FAIL(); /* Nothing fired twice. */ + if (timeouts[i] <= now) { + if (!(fired[i] || deleted[i])) + FAIL(); + } else { + if (fired[i]) + FAIL(); + } + if (fired[i] && deleted[i]) + FAIL(); + if (cfg->finalize > 1) { + if (!fired[i]) + timeout_del(&t[i]); + } + } + + /* Now nothing more should fire between now and the end of time. */ + if (cfg->finalize) { + timeouts_update(tos, THE_END_OF_TIME); + if (cfg->finalize > 1) { + if (timeouts_get(tos)) + FAIL(); + TIMEOUTS_FOREACH(to, tos, TIMEOUTS_ALL) + FAIL(); + } + } + rv = 0; + + done: + if (tos) timeouts_close(tos); + if (t) free(t); + if (timeouts) free(timeouts); + if (fired) free(fired); + if (found) free(found); + if (deleted) free(deleted); + return rv; +} + +struct intervals_cfg { + const timeout_t *timeouts; + int n_timeouts; + timeout_t start_at; + timeout_t end_at; + timeout_t skip; +}; + +int +check_intervals(struct intervals_cfg *cfg) +{ + int i, err; + int rv = 1; + struct timeout *to; + struct timeout *t = calloc(cfg->n_timeouts, sizeof(struct timeout)); + unsigned *fired = calloc(cfg->n_timeouts, sizeof(unsigned)); + struct timeouts *tos = timeouts_open(0, &err); + + timeout_t now = cfg->start_at; + if (!t || !tos || !fired) + FAIL(); + + timeouts_update(tos, now); + + for (i = 0; i < cfg->n_timeouts; ++i) { + if (&t[i] != timeout_init(&t[i], TIMEOUT_INT)) + FAIL(); + if (timeout_pending(&t[i])) + FAIL(); + if (timeout_expired(&t[i])) + FAIL(); + + timeouts_add(tos, &t[i], cfg->timeouts[i]); + if (! timeout_pending(&t[i])) + FAIL(); + if (timeout_expired(&t[i])) + FAIL(); + } + + while (now < cfg->end_at) { + timeout_t delay = timeouts_timeout(tos); + if (cfg->skip && delay < cfg->skip) + delay = cfg->skip; + timeouts_step(tos, delay); + now += delay; + + while (NULL != (to = timeouts_get(tos))) { + i = to - &t[0]; + assert(&t[i] == to); + fired[i]++; + if (0 != (to->expires - cfg->start_at) % cfg->timeouts[i]) + FAIL(); + if (to->expires <= now) + FAIL(); + if (to->expires > now + cfg->timeouts[i]) + FAIL(); + } + if (!timeouts_check(tos, stderr)) + FAIL(); + } + + timeout_t duration = now - cfg->start_at; + for (i = 0; i < cfg->n_timeouts; ++i) { + if (cfg->skip) { + if (fired[i] > duration / cfg->timeouts[i]) + FAIL(); + } else { + if (fired[i] != duration / cfg->timeouts[i]) + FAIL(); + } + if (!timeout_pending(&t[i])) + FAIL(); + } + + rv = 0; + done: + if (t) free(t); + if (fired) free(fired); + if (tos) free(tos); + return rv; +} + +int +main(int argc, char **argv) +{ + int j; + int n_failed = 0; +#define DO(fn) do { \ + printf("."); fflush(stdout); \ + if (fn) { \ + ++n_failed; \ + printf("%s failed\n", #fn); \ + } \ + } while (0) + +#define DO_N(n, fn) do { \ + for (j = 0; j < (n); ++j) { \ + DO(fn); \ + } \ + } while (0) + + DO(check_misc()); + DO(check_open_close(1000, 1000)); + DO(check_open_close(0, TIMEOUT_mHZ)); + + struct rand_cfg cfg1 = { + .min_timeout = 1, + .max_timeout = 100, + .start_at = 5, + .end_at = 1000, + .n_timeouts = 1000, + .max_step = 10, + .relative = 0, + .try_removing = 0, + .finalize = 2, + }; + DO_N(300,check_randomized(&cfg1)); + + struct rand_cfg cfg2 = { + .min_timeout = 20, + .max_timeout = 1000, + .start_at = 10, + .end_at = 100, + .n_timeouts = 1000, + .max_step = 5, + .relative = 1, + .try_removing = 0, + .finalize = 2, + }; + DO_N(300,check_randomized(&cfg2)); + + struct rand_cfg cfg2b = { + .min_timeout = 20, + .max_timeout = 1000, + .start_at = 10, + .end_at = 100, + .n_timeouts = 1000, + .max_step = 5, + .relative = 1, + .try_removing = 0, + .finalize = 1, + }; + DO_N(300,check_randomized(&cfg2b)); + + struct rand_cfg cfg2c = { + .min_timeout = 20, + .max_timeout = 1000, + .start_at = 10, + .end_at = 100, + .n_timeouts = 1000, + .max_step = 5, + .relative = 1, + .try_removing = 0, + .finalize = 0, + }; + DO_N(300,check_randomized(&cfg2c)); + + struct rand_cfg cfg3 = { + .min_timeout = 2000, + .max_timeout = ((uint64_t)1) << 50, + .start_at = 100, + .end_at = ((uint64_t)1) << 49, + .n_timeouts = 1000, + .max_step = 1<<31, + .relative = 0, + .try_removing = 0, + .finalize = 2, + }; + DO_N(10,check_randomized(&cfg3)); + + struct rand_cfg cfg3b = { + .min_timeout = ((uint64_t)1) << 50, + .max_timeout = ((uint64_t)1) << 52, + .start_at = 100, + .end_at = ((uint64_t)1) << 53, + .n_timeouts = 1000, + .max_step = ((uint64_t)1)<<48, + .relative = 0, + .try_removing = 0, + .finalize = 2, + }; + DO_N(10,check_randomized(&cfg3b)); + + struct rand_cfg cfg4 = { + .min_timeout = 2000, + .max_timeout = ((uint64_t)1) << 30, + .start_at = 100, + .end_at = ((uint64_t)1) << 26, + .n_timeouts = 10000, + .max_step = 1<<16, + .relative = 0, + .try_removing = 3, + .finalize = 2, + }; + DO_N(10,check_randomized(&cfg4)); + + const timeout_t primes[] = { + 2,3,5,7,11,13,17,19,23,29,31,37,41,43,47,53, + 59,61,67,71,73,79,83,89,97 + }; + const timeout_t factors_of_1337[] = { + 1, 7, 191, 1337 + }; + const timeout_t multiples_of_five[] = { + 5, 10, 15, 20, 25, 30, 35, 40, 45, 50 + }; + + struct intervals_cfg icfg1 = { + .timeouts = primes, + .n_timeouts = sizeof(primes)/sizeof(timeout_t), + .start_at = 50, + .end_at = 5322, + .skip = 0, + }; + DO(check_intervals(&icfg1)); + + struct intervals_cfg icfg2 = { + .timeouts = factors_of_1337, + .n_timeouts = sizeof(factors_of_1337)/sizeof(timeout_t), + .start_at = 50, + .end_at = 50000, + .skip = 0, + }; + DO(check_intervals(&icfg2)); + + struct intervals_cfg icfg3 = { + .timeouts = multiples_of_five, + .n_timeouts = sizeof(multiples_of_five)/sizeof(timeout_t), + .start_at = 49, + .end_at = 5333, + .skip = 0, + }; + DO(check_intervals(&icfg3)); + + struct intervals_cfg icfg4 = { + .timeouts = primes, + .n_timeouts = sizeof(primes)/sizeof(timeout_t), + .start_at = 50, + .end_at = 5322, + .skip = 16, + }; + DO(check_intervals(&icfg4)); + + if (n_failed) { + puts("\nFAIL"); + } else { + puts("\nOK"); + } + return !!n_failed; +} + +/* TODO: + + * Solve PR#3. + + * Investigate whether any untaken branches are possible. + + */ diff --git a/src/ext/timeouts/timeout-bitops.c b/src/ext/timeouts/timeout-bitops.c new file mode 100644 index 0000000000..a018f33b95 --- /dev/null +++ b/src/ext/timeouts/timeout-bitops.c @@ -0,0 +1,254 @@ +#include <stdint.h> +#include <limits.h> +#ifdef _MSC_VER +#include <intrin.h> /* _BitScanForward, _BitScanReverse */ +#endif + +/* First define ctz and clz functions; these are compiler-dependent if + * you want them to be fast. */ +#if defined(__GNUC__) && !defined(TIMEOUT_DISABLE_GNUC_BITOPS) + +#ifndef LONG_BIT +#define LONG_BIT (SIZEOF_LONG*CHAR_BIT) +#endif + +/* On GCC and clang and some others, we can use __builtin functions. They + * are not defined for n==0, but timeout.s never calls them with n==0. */ + +#define ctz64(n) __builtin_ctzll(n) +#define clz64(n) __builtin_clzll(n) +#if LONG_BIT == 32 +#define ctz32(n) __builtin_ctzl(n) +#define clz32(n) __builtin_clzl(n) +#else +#define ctz32(n) __builtin_ctz(n) +#define clz32(n) __builtin_clz(n) +#endif + +#elif defined(_MSC_VER) && !defined(TIMEOUT_DISABLE_MSVC_BITOPS) + +/* On MSVC, we have these handy functions. We can ignore their return + * values, since we will never supply val == 0. */ + +static __inline int ctz32(unsigned long val) +{ + DWORD zeros = 0; + _BitScanForward(&zeros, val); + return zeros; +} +static __inline int clz32(unsigned long val) +{ + DWORD zeros = 0; + _BitScanReverse(&zeros, val); + return zeros; +} +#ifdef _WIN64 +/* According to the documentation, these only exist on Win64. */ +static __inline int ctz64(uint64_t val) +{ + DWORD zeros = 0; + _BitScanForward64(&zeros, val); + return zeros; +} +static __inline int clz64(uint64_t val) +{ + DWORD zeros = 0; + _BitScanReverse64(&zeros, val); + return zeros; +} +#else +static __inline int ctz64(uint64_t val) +{ + uint32_t lo = (uint32_t) val; + uint32_t hi = (uint32_t) (val >> 32); + return lo ? ctz32(lo) : 32 + ctz32(hi); +} +static __inline int clz64(uint64_t val) +{ + uint32_t lo = (uint32_t) val; + uint32_t hi = (uint32_t) (val >> 32); + return hi ? clz32(hi) : 32 + clz32(lo); +} +#endif + +/* End of MSVC case. */ + +#else + +/* TODO: There are more clever ways to do this in the generic case. */ + + +#define process_(one, cz_bits, bits) \ + if (x < ( one << (cz_bits - bits))) { rv += bits; x <<= bits; } + +#define process64(bits) process_((UINT64_C(1)), 64, (bits)) +static inline int clz64(uint64_t x) +{ + int rv = 0; + + process64(32); + process64(16); + process64(8); + process64(4); + process64(2); + process64(1); + return rv; +} +#define process32(bits) process_((UINT32_C(1)), 32, (bits)) +static inline int clz32(uint32_t x) +{ + int rv = 0; + + process32(16); + process32(8); + process32(4); + process32(2); + process32(1); + return rv; +} + +#undef process_ +#undef process32 +#undef process64 +#define process_(one, bits) \ + if ((x & ((one << (bits))-1)) == 0) { rv += bits; x >>= bits; } + +#define process64(bits) process_((UINT64_C(1)), bits) +static inline int ctz64(uint64_t x) +{ + int rv = 0; + + process64(32); + process64(16); + process64(8); + process64(4); + process64(2); + process64(1); + return rv; +} + +#define process32(bits) process_((UINT32_C(1)), bits) +static inline int ctz32(uint32_t x) +{ + int rv = 0; + + process32(16); + process32(8); + process32(4); + process32(2); + process32(1); + return rv; +} + +#undef process32 +#undef process64 +#undef process_ + +/* End of generic case */ + +#endif /* End of defining ctz */ + +#ifdef TEST_BITOPS +#include <stdio.h> +#include <stdlib.h> + +static uint64_t testcases[] = { + 13371337 * 10, + 100, + 385789752, + 82574, + (((uint64_t)1)<<63) + (((uint64_t)1)<<31) + 10101 +}; + +static int +naive_clz(int bits, uint64_t v) +{ + int r = 0; + uint64_t bit = ((uint64_t)1) << (bits-1); + while (bit && 0 == (v & bit)) { + r++; + bit >>= 1; + } + /* printf("clz(%d,%lx) -> %d\n", bits, v, r); */ + return r; +} + +static int +naive_ctz(int bits, uint64_t v) +{ + int r = 0; + uint64_t bit = 1; + while (bit && 0 == (v & bit)) { + r++; + bit <<= 1; + if (r == bits) + break; + } + /* printf("ctz(%d,%lx) -> %d\n", bits, v, r); */ + return r; +} + +static int +check(uint64_t vv) +{ + uint32_t v32 = (uint32_t) vv; + + if (vv == 0) + return 1; /* c[tl]z64(0) is undefined. */ + + if (ctz64(vv) != naive_ctz(64, vv)) { + printf("mismatch with ctz64: %d\n", ctz64(vv)); + exit(1); + return 0; + } + if (clz64(vv) != naive_clz(64, vv)) { + printf("mismatch with clz64: %d\n", clz64(vv)); + exit(1); + return 0; + } + + if (v32 == 0) + return 1; /* c[lt]z(0) is undefined. */ + + if (ctz32(v32) != naive_ctz(32, v32)) { + printf("mismatch with ctz32: %d\n", ctz32(v32)); + exit(1); + return 0; + } + if (clz32(v32) != naive_clz(32, v32)) { + printf("mismatch with clz32: %d\n", clz32(v32)); + exit(1); + return 0; + } + return 1; +} + +int +main(int c, char **v) +{ + unsigned int i; + const unsigned int n = sizeof(testcases)/sizeof(testcases[0]); + int result = 0; + + for (i = 0; i <= 63; ++i) { + uint64_t x = 1 << i; + if (!check(x)) + result = 1; + --x; + if (!check(x)) + result = 1; + } + + for (i = 0; i < n; ++i) { + if (! check(testcases[i])) + result = 1; + } + if (result) { + puts("FAIL"); + } else { + puts("OK"); + } + return result; +} +#endif + diff --git a/src/ext/timeouts/timeout-debug.h b/src/ext/timeouts/timeout-debug.h new file mode 100644 index 0000000000..fc727a6b42 --- /dev/null +++ b/src/ext/timeouts/timeout-debug.h @@ -0,0 +1,77 @@ +/* + * D E B U G R O U T I N E S + * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#if TIMEOUT_DEBUG - 0 +#include <stdlib.h> +#include <stdio.h> + +#undef TIMEOUT_DEBUG +#define TIMEOUT_DEBUG 1 +#define DEBUG_LEVEL timeout_debug + +static int timeout_debug; + +#define SAYit_(lvl, fmt, ...) do { \ + if (DEBUG_LEVEL >= (lvl)) \ + fprintf(stderr, fmt "%s", __FILE__, __LINE__, __func__, __VA_ARGS__); \ +} while (0) + +#define SAYit(lvl, ...) SAYit_((lvl), "%s:%d:%s: " __VA_ARGS__, "\n") + +#define PANIC(...) do { \ + SAYit(0, __VA_ARGS__); \ + _Exit(EXIT_FAILURE); \ +} while (0) +#else +#undef TIMEOUT_DEBUG +#define TIMEOUT_DEBUG 0 +#define DEBUG_LEVEL 0 + +#define SAYit(...) (void)0 +#endif + +#define SAY(...) SAYit(1, __VA_ARGS__) +#define HAI SAY("HAI") + + +static inline char *fmt_(char *buf, uint64_t ts, int wheel_bit, int wheel_num) { + char *p = buf; + int wheel, n, i; + + for (wheel = wheel_num - 2; wheel >= 0; wheel--) { + n = ((1 << wheel_bit) - 1) & (ts >> (wheel * WHEEL_BIT)); + + for (i = wheel_bit - 1; i >= 0; i--) { + *p++ = '0' + !!(n & (1 << i)); + } + + if (wheel != 0) + *p++ = ':'; + } + + *p = 0; + + return buf; +} /* fmt_() */ + +#define fmt(ts) fmt_(((char[((1 << WHEEL_BIT) * WHEEL_NUM) + WHEEL_NUM + 1]){ 0 }), (ts), WHEEL_BIT, WHEEL_NUM) + + +static inline char *bin64_(char *buf, uint64_t n, int wheel_bit) { + char *p = buf; + int i; + + for (i = 0; i < (1 << wheel_bit); i++) { + *p++ = "01"[0x1 & (n >> (((1 << wheel_bit) - 1) - i))]; + } + + *p = 0; + + return buf; +} /* bin64_() */ + +#define bin64(ts) bin64_(((char[((1 << WHEEL_BIT) * WHEEL_NUM) + 1]){ 0 }), (ts), WHEEL_BIT) + + diff --git a/src/ext/timeouts/timeout.c b/src/ext/timeouts/timeout.c new file mode 100644 index 0000000000..bd463a700d --- /dev/null +++ b/src/ext/timeouts/timeout.c @@ -0,0 +1,754 @@ +/* ========================================================================== + * timeout.c - Tickless hierarchical timing wheel. + * -------------------------------------------------------------------------- + * Copyright (c) 2013, 2014 William Ahern + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the + * following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * ========================================================================== + */ +#ifdef HAVE_CONFIG_H +#include "orconfig.h" +#endif +#include <limits.h> /* CHAR_BIT */ + +#include <stddef.h> /* NULL */ +#include <stdlib.h> /* malloc(3) free(3) */ +#include <stdio.h> /* FILE fprintf(3) */ + +#include <inttypes.h> /* UINT64_C uint64_t */ + +#include <string.h> /* memset(3) */ + +#include <errno.h> /* errno */ + +#include "tor_queue.h" /* TAILQ(3) */ + +#include "timeout.h" + +#ifndef TIMEOUT_DEBUG +#define TIMEOUT_DEBUG 0 +#endif + +#if TIMEOUT_DEBUG - 0 +#include "timeout-debug.h" +#endif + +#ifdef TIMEOUT_DISABLE_RELATIVE_ACCESS +#define TO_SET_TIMEOUTS(to, T) ((void)0) +#else +#define TO_SET_TIMEOUTS(to, T) ((to)->timeouts = (T)) +#endif + +/* + * A N C I L L A R Y R O U T I N E S + * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#define abstime_t timeout_t /* for documentation purposes */ +#define reltime_t timeout_t /* "" */ + +#if !defined countof +#define countof(a) (sizeof (a) / sizeof *(a)) +#endif + +#if !defined endof +#define endof(a) (&(a)[countof(a)]) +#endif + +#if !defined MIN +#define MIN(a, b) (((a) < (b))? (a) : (b)) +#endif + +#if !defined MAX +#define MAX(a, b) (((a) > (b))? (a) : (b)) +#endif + +#if !defined TOR_TAILQ_CONCAT +#define TOR_TAILQ_CONCAT(head1, head2, field) do { \ + if (!TOR_TAILQ_EMPTY(head2)) { \ + *(head1)->tqh_last = (head2)->tqh_first; \ + (head2)->tqh_first->field.tqe_prev = (head1)->tqh_last; \ + (head1)->tqh_last = (head2)->tqh_last; \ + TOR_TAILQ_INIT((head2)); \ + } \ +} while (0) +#endif + +#if !defined TOR_TAILQ_FOREACH_SAFE +#define TOR_TAILQ_FOREACH_SAFE(var, head, field, tvar) \ + for ((var) = TOR_TAILQ_FIRST(head); \ + (var) && ((tvar) = TOR_TAILQ_NEXT(var, field), 1); \ + (var) = (tvar)) +#endif + + +/* + * B I T M A N I P U L A T I O N R O U T I N E S + * + * The macros and routines below implement wheel parameterization. The + * inputs are: + * + * WHEEL_BIT - The number of value bits mapped in each wheel. The + * lowest-order WHEEL_BIT bits index the lowest-order (highest + * resolution) wheel, the next group of WHEEL_BIT bits the + * higher wheel, etc. + * + * WHEEL_NUM - The number of wheels. WHEEL_BIT * WHEEL_NUM = the number of + * value bits used by all the wheels. For the default of 6 and + * 4, only the low 24 bits are processed. Any timeout value + * larger than this will cycle through again. + * + * The implementation uses bit fields to remember which slot in each wheel + * is populated, and to generate masks of expiring slots according to the + * current update interval (i.e. the "tickless" aspect). The slots to + * process in a wheel are (populated-set & interval-mask). + * + * WHEEL_BIT cannot be larger than 6 bits because 2^6 -> 64 is the largest + * number of slots which can be tracked in a uint64_t integer bit field. + * WHEEL_BIT cannot be smaller than 3 bits because of our rotr and rotl + * routines, which only operate on all the value bits in an integer, and + * there's no integer smaller than uint8_t. + * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#if !defined WHEEL_BIT +#define WHEEL_BIT 6 +#endif + +#if !defined WHEEL_NUM +#define WHEEL_NUM 4 +#endif + +#define WHEEL_LEN (1U << WHEEL_BIT) +#define WHEEL_MAX (WHEEL_LEN - 1) +#define WHEEL_MASK (WHEEL_LEN - 1) +#define TIMEOUT_MAX ((TIMEOUT_C(1) << (WHEEL_BIT * WHEEL_NUM)) - 1) + +#include "timeout-bitops.c" + +#if WHEEL_BIT == 6 +#define ctz(n) ctz64(n) +#define clz(n) clz64(n) +#define fls(n) ((int)(64 - clz64(n))) +#else +#define ctz(n) ctz32(n) +#define clz(n) clz32(n) +#define fls(n) ((int)(32 - clz32(n))) +#endif + +#if WHEEL_BIT == 6 +#define WHEEL_C(n) UINT64_C(n) +#define WHEEL_PRIu PRIu64 +#define WHEEL_PRIx PRIx64 + +typedef uint64_t wheel_t; + +#elif WHEEL_BIT == 5 + +#define WHEEL_C(n) UINT32_C(n) +#define WHEEL_PRIu PRIu32 +#define WHEEL_PRIx PRIx32 + +typedef uint32_t wheel_t; + +#elif WHEEL_BIT == 4 + +#define WHEEL_C(n) UINT16_C(n) +#define WHEEL_PRIu PRIu16 +#define WHEEL_PRIx PRIx16 + +typedef uint16_t wheel_t; + +#elif WHEEL_BIT == 3 + +#define WHEEL_C(n) UINT8_C(n) +#define WHEEL_PRIu PRIu8 +#define WHEEL_PRIx PRIx8 + +typedef uint8_t wheel_t; + +#else +#error invalid WHEEL_BIT value +#endif + + +static inline wheel_t rotl(const wheel_t v, int c) { + if (!(c &= (sizeof v * CHAR_BIT - 1))) + return v; + + return (v << c) | (v >> (sizeof v * CHAR_BIT - c)); +} /* rotl() */ + + +static inline wheel_t rotr(const wheel_t v, int c) { + if (!(c &= (sizeof v * CHAR_BIT - 1))) + return v; + + return (v >> c) | (v << (sizeof v * CHAR_BIT - c)); +} /* rotr() */ + + +/* + * T I M E R R O U T I N E S + * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +TOR_TAILQ_HEAD(timeout_list, timeout); + +struct timeouts { + struct timeout_list wheel[WHEEL_NUM][WHEEL_LEN], expired; + + wheel_t pending[WHEEL_NUM]; + + timeout_t curtime; + timeout_t hertz; +}; /* struct timeouts */ + + +static struct timeouts *timeouts_init(struct timeouts *T, timeout_t hz) { + unsigned i, j; + + for (i = 0; i < countof(T->wheel); i++) { + for (j = 0; j < countof(T->wheel[i]); j++) { + TOR_TAILQ_INIT(&T->wheel[i][j]); + } + } + + TOR_TAILQ_INIT(&T->expired); + + for (i = 0; i < countof(T->pending); i++) { + T->pending[i] = 0; + } + + T->curtime = 0; + T->hertz = (hz)? hz : TIMEOUT_mHZ; + + return T; +} /* timeouts_init() */ + + +TIMEOUT_PUBLIC struct timeouts *timeouts_open(timeout_t hz, int *error) { + struct timeouts *T; + + if ((T = malloc(sizeof *T))) + return timeouts_init(T, hz); + + *error = errno; + + return NULL; +} /* timeouts_open() */ + + +static void timeouts_reset(struct timeouts *T) { + struct timeout_list reset; + struct timeout *to; + unsigned i, j; + + TOR_TAILQ_INIT(&reset); + + for (i = 0; i < countof(T->wheel); i++) { + for (j = 0; j < countof(T->wheel[i]); j++) { + TOR_TAILQ_CONCAT(&reset, &T->wheel[i][j], tqe); + } + } + + TOR_TAILQ_CONCAT(&reset, &T->expired, tqe); + + TOR_TAILQ_FOREACH(to, &reset, tqe) { + to->pending = NULL; + TO_SET_TIMEOUTS(to, NULL); + } +} /* timeouts_reset() */ + + +TIMEOUT_PUBLIC void timeouts_close(struct timeouts *T) { + /* + * NOTE: Delete installed timeouts so timeout_pending() and + * timeout_expired() worked as expected. + */ + timeouts_reset(T); + + free(T); +} /* timeouts_close() */ + + +TIMEOUT_PUBLIC timeout_t timeouts_hz(struct timeouts *T) { + return T->hertz; +} /* timeouts_hz() */ + + +TIMEOUT_PUBLIC void timeouts_del(struct timeouts *T, struct timeout *to) { + if (to->pending) { + TOR_TAILQ_REMOVE(to->pending, to, tqe); + + if (to->pending != &T->expired && TOR_TAILQ_EMPTY(to->pending)) { + ptrdiff_t index = to->pending - &T->wheel[0][0]; + int wheel = (int) (index / WHEEL_LEN); + int slot = index % WHEEL_LEN; + + T->pending[wheel] &= ~(WHEEL_C(1) << slot); + } + + to->pending = NULL; + TO_SET_TIMEOUTS(to, NULL); + } +} /* timeouts_del() */ + + +static inline reltime_t timeout_rem(struct timeouts *T, struct timeout *to) { + return to->expires - T->curtime; +} /* timeout_rem() */ + + +static inline int timeout_wheel(timeout_t timeout) { + /* must be called with timeout != 0, so fls input is nonzero */ + return (fls(MIN(timeout, TIMEOUT_MAX)) - 1) / WHEEL_BIT; +} /* timeout_wheel() */ + + +static inline int timeout_slot(int wheel, timeout_t expires) { + return WHEEL_MASK & ((expires >> (wheel * WHEEL_BIT)) - !!wheel); +} /* timeout_slot() */ + + +static void timeouts_sched(struct timeouts *T, struct timeout *to, timeout_t expires) { + timeout_t rem; + int wheel, slot; + + timeouts_del(T, to); + + to->expires = expires; + + TO_SET_TIMEOUTS(to, T); + + if (expires > T->curtime) { + rem = timeout_rem(T, to); + + /* rem is nonzero since: + * rem == timeout_rem(T,to), + * == to->expires - T->curtime + * and above we have expires > T->curtime. + */ + wheel = timeout_wheel(rem); + slot = timeout_slot(wheel, to->expires); + + to->pending = &T->wheel[wheel][slot]; + TOR_TAILQ_INSERT_TAIL(to->pending, to, tqe); + + T->pending[wheel] |= WHEEL_C(1) << slot; + } else { + to->pending = &T->expired; + TOR_TAILQ_INSERT_TAIL(to->pending, to, tqe); + } +} /* timeouts_sched() */ + + +#ifndef TIMEOUT_DISABLE_INTERVALS +static void timeouts_readd(struct timeouts *T, struct timeout *to) { + to->expires += to->interval; + + if (to->expires <= T->curtime) { + /* If we've missed the next firing of this timeout, reschedule + * it to occur at the next multiple of its interval after + * the last time that it fired. + */ + timeout_t n = T->curtime - to->expires; + timeout_t r = n % to->interval; + to->expires = T->curtime + (to->interval - r); + } + + timeouts_sched(T, to, to->expires); +} /* timeouts_readd() */ +#endif + + +TIMEOUT_PUBLIC void timeouts_add(struct timeouts *T, struct timeout *to, timeout_t timeout) { +#ifndef TIMEOUT_DISABLE_INTERVALS + if (to->flags & TIMEOUT_INT) + to->interval = MAX(1, timeout); +#endif + + if (to->flags & TIMEOUT_ABS) + timeouts_sched(T, to, timeout); + else + timeouts_sched(T, to, T->curtime + timeout); +} /* timeouts_add() */ + + +TIMEOUT_PUBLIC void timeouts_update(struct timeouts *T, abstime_t curtime) { + timeout_t elapsed = curtime - T->curtime; + struct timeout_list todo; + int wheel; + + TOR_TAILQ_INIT(&todo); + + /* + * There's no avoiding looping over every wheel. It's best to keep + * WHEEL_NUM smallish. + */ + for (wheel = 0; wheel < WHEEL_NUM; wheel++) { + wheel_t pending; + + /* + * Calculate the slots expiring in this wheel + * + * If the elapsed time is greater than the maximum period of + * the wheel, mark every position as expiring. + * + * Otherwise, to determine the expired slots fill in all the + * bits between the last slot processed and the current + * slot, inclusive of the last slot. We'll bitwise-AND this + * with our pending set below. + * + * If a wheel rolls over, force a tick of the next higher + * wheel. + */ + if ((elapsed >> (wheel * WHEEL_BIT)) > WHEEL_MAX) { + pending = (wheel_t)~WHEEL_C(0); + } else { + wheel_t _elapsed = WHEEL_MASK & (elapsed >> (wheel * WHEEL_BIT)); + int oslot, nslot; + + /* + * TODO: It's likely that at least one of the + * following three bit fill operations is redundant + * or can be replaced with a simpler operation. + */ + oslot = WHEEL_MASK & (T->curtime >> (wheel * WHEEL_BIT)); + pending = rotl(((UINT64_C(1) << _elapsed) - 1), oslot); + + nslot = WHEEL_MASK & (curtime >> (wheel * WHEEL_BIT)); + pending |= rotr(rotl(((WHEEL_C(1) << _elapsed) - 1), nslot), (int)_elapsed); + pending |= WHEEL_C(1) << nslot; + } + + while (pending & T->pending[wheel]) { + /* ctz input cannot be zero: loop condition. */ + int slot = ctz(pending & T->pending[wheel]); + TOR_TAILQ_CONCAT(&todo, &T->wheel[wheel][slot], tqe); + T->pending[wheel] &= ~(UINT64_C(1) << slot); + } + + if (!(0x1 & pending)) + break; /* break if we didn't wrap around end of wheel */ + + /* if we're continuing, the next wheel must tick at least once */ + elapsed = MAX(elapsed, (WHEEL_LEN << (wheel * WHEEL_BIT))); + } + + T->curtime = curtime; + + while (!TOR_TAILQ_EMPTY(&todo)) { + struct timeout *to = TOR_TAILQ_FIRST(&todo); + + TOR_TAILQ_REMOVE(&todo, to, tqe); + to->pending = NULL; + + timeouts_sched(T, to, to->expires); + } + + return; +} /* timeouts_update() */ + +TIMEOUT_PUBLIC timeout_t timeouts_get_curtime(struct timeouts *T) { + return T->curtime; +} /* timeouts_get_curtime() */ + +TIMEOUT_PUBLIC void timeouts_step(struct timeouts *T, reltime_t elapsed) { + timeouts_update(T, T->curtime + elapsed); +} /* timeouts_step() */ + + +TIMEOUT_PUBLIC bool timeouts_pending(struct timeouts *T) { + wheel_t pending = 0; + int wheel; + + for (wheel = 0; wheel < WHEEL_NUM; wheel++) { + pending |= T->pending[wheel]; + } + + return !!pending; +} /* timeouts_pending() */ + + +TIMEOUT_PUBLIC bool timeouts_expired(struct timeouts *T) { + return !TOR_TAILQ_EMPTY(&T->expired); +} /* timeouts_expired() */ + + +/* + * Calculate the interval before needing to process any timeouts pending on + * any wheel. + * + * (This is separated from the public API routine so we can evaluate our + * wheel invariant assertions irrespective of the expired queue.) + * + * This might return a timeout value sooner than any installed timeout if + * only higher-order wheels have timeouts pending. We can only know when to + * process a wheel, not precisely when a timeout is scheduled. Our timeout + * accuracy could be off by 2^(N*M)-1 units where N is the wheel number and + * M is WHEEL_BIT. Only timeouts which have fallen through to wheel 0 can be + * known exactly. + * + * We should never return a timeout larger than the lowest actual timeout. + */ +static timeout_t timeouts_int(struct timeouts *T) { + timeout_t timeout = ~TIMEOUT_C(0), _timeout; + timeout_t relmask; + int wheel, slot; + + relmask = 0; + + for (wheel = 0; wheel < WHEEL_NUM; wheel++) { + if (T->pending[wheel]) { + slot = WHEEL_MASK & (T->curtime >> (wheel * WHEEL_BIT)); + + /* ctz input cannot be zero: T->pending[wheel] is + * nonzero, so rotr() is nonzero. */ + _timeout = (ctz(rotr(T->pending[wheel], slot)) + !!wheel) << (wheel * WHEEL_BIT); + /* +1 to higher order wheels as those timeouts are one rotation in the future (otherwise they'd be on a lower wheel or expired) */ + + _timeout -= relmask & T->curtime; + /* reduce by how much lower wheels have progressed */ + + timeout = MIN(_timeout, timeout); + } + + relmask <<= WHEEL_BIT; + relmask |= WHEEL_MASK; + } + + return timeout; +} /* timeouts_int() */ + + +/* + * Calculate the interval our caller can wait before needing to process + * events. + */ +TIMEOUT_PUBLIC timeout_t timeouts_timeout(struct timeouts *T) { + if (!TOR_TAILQ_EMPTY(&T->expired)) + return 0; + + return timeouts_int(T); +} /* timeouts_timeout() */ + + +TIMEOUT_PUBLIC struct timeout *timeouts_get(struct timeouts *T) { + if (!TOR_TAILQ_EMPTY(&T->expired)) { + struct timeout *to = TOR_TAILQ_FIRST(&T->expired); + + TOR_TAILQ_REMOVE(&T->expired, to, tqe); + to->pending = NULL; + TO_SET_TIMEOUTS(to, NULL); + +#ifndef TIMEOUT_DISABLE_INTERVALS + if ((to->flags & TIMEOUT_INT) && to->interval > 0) + timeouts_readd(T, to); +#endif + + return to; + } else { + return 0; + } +} /* timeouts_get() */ + + +/* + * Use dumb looping to locate the earliest timeout pending on the wheel so + * our invariant assertions can check the result of our optimized code. + */ +static struct timeout *timeouts_min(struct timeouts *T) { + struct timeout *to, *min = NULL; + unsigned i, j; + + for (i = 0; i < countof(T->wheel); i++) { + for (j = 0; j < countof(T->wheel[i]); j++) { + TOR_TAILQ_FOREACH(to, &T->wheel[i][j], tqe) { + if (!min || to->expires < min->expires) + min = to; + } + } + } + + return min; +} /* timeouts_min() */ + + +/* + * Check some basic algorithm invariants. If these invariants fail then + * something is definitely broken. + */ +#define report(...) do { \ + if ((fp)) \ + fprintf(fp, __VA_ARGS__); \ +} while (0) + +#define check(expr, ...) do { \ + if (!(expr)) { \ + report(__VA_ARGS__); \ + return 0; \ + } \ +} while (0) + +TIMEOUT_PUBLIC bool timeouts_check(struct timeouts *T, FILE *fp) { + timeout_t timeout; + struct timeout *to; + + if ((to = timeouts_min(T))) { + check(to->expires > T->curtime, "missed timeout (expires:%" TIMEOUT_PRIu " <= curtime:%" TIMEOUT_PRIu ")\n", to->expires, T->curtime); + + timeout = timeouts_int(T); + check(timeout <= to->expires - T->curtime, "wrong soft timeout (soft:%" TIMEOUT_PRIu " > hard:%" TIMEOUT_PRIu ") (expires:%" TIMEOUT_PRIu "; curtime:%" TIMEOUT_PRIu ")\n", timeout, (to->expires - T->curtime), to->expires, T->curtime); + + timeout = timeouts_timeout(T); + check(timeout <= to->expires - T->curtime, "wrong soft timeout (soft:%" TIMEOUT_PRIu " > hard:%" TIMEOUT_PRIu ") (expires:%" TIMEOUT_PRIu "; curtime:%" TIMEOUT_PRIu ")\n", timeout, (to->expires - T->curtime), to->expires, T->curtime); + } else { + timeout = timeouts_timeout(T); + + if (!TOR_TAILQ_EMPTY(&T->expired)) + check(timeout == 0, "wrong soft timeout (soft:%" TIMEOUT_PRIu " != hard:%" TIMEOUT_PRIu ")\n", timeout, TIMEOUT_C(0)); + else + check(timeout == ~TIMEOUT_C(0), "wrong soft timeout (soft:%" TIMEOUT_PRIu " != hard:%" TIMEOUT_PRIu ")\n", timeout, ~TIMEOUT_C(0)); + } + + return 1; +} /* timeouts_check() */ + + +#define ENTER \ + do { \ + static const int pc0 = __LINE__; \ + switch (pc0 + it->pc) { \ + case __LINE__: (void)0 + +#define SAVE_AND_DO(do_statement) \ + do { \ + it->pc = __LINE__ - pc0; \ + do_statement; \ + case __LINE__: (void)0; \ + } while (0) + +#define YIELD(rv) \ + SAVE_AND_DO(return (rv)) + +#define LEAVE \ + SAVE_AND_DO(break); \ + } \ + } while (0) + +TIMEOUT_PUBLIC struct timeout *timeouts_next(struct timeouts *T, struct timeouts_it *it) { + struct timeout *to; + + ENTER; + + if (it->flags & TIMEOUTS_EXPIRED) { + if (it->flags & TIMEOUTS_CLEAR) { + while ((to = timeouts_get(T))) { + YIELD(to); + } + } else { + TOR_TAILQ_FOREACH_SAFE(to, &T->expired, tqe, it->to) { + YIELD(to); + } + } + } + + if (it->flags & TIMEOUTS_PENDING) { + for (it->i = 0; it->i < countof(T->wheel); it->i++) { + for (it->j = 0; it->j < countof(T->wheel[it->i]); it->j++) { + TOR_TAILQ_FOREACH_SAFE(to, &T->wheel[it->i][it->j], tqe, it->to) { + YIELD(to); + } + } + } + } + + LEAVE; + + return NULL; +} /* timeouts_next */ + +#undef LEAVE +#undef YIELD +#undef SAVE_AND_DO +#undef ENTER + + +/* + * T I M E O U T R O U T I N E S + * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +TIMEOUT_PUBLIC struct timeout *timeout_init(struct timeout *to, int flags) { + memset(to, 0, sizeof *to); + + to->flags = flags; + + return to; +} /* timeout_init() */ + + +#ifndef TIMEOUT_DISABLE_RELATIVE_ACCESS +TIMEOUT_PUBLIC bool timeout_pending(struct timeout *to) { + return to->pending && to->pending != &to->timeouts->expired; +} /* timeout_pending() */ + + +TIMEOUT_PUBLIC bool timeout_expired(struct timeout *to) { + return to->pending && to->pending == &to->timeouts->expired; +} /* timeout_expired() */ + + +TIMEOUT_PUBLIC void timeout_del(struct timeout *to) { + timeouts_del(to->timeouts, to); +} /* timeout_del() */ +#endif + + +/* + * V E R S I O N I N T E R F A C E S + * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +TIMEOUT_PUBLIC int timeout_version(void) { + return TIMEOUT_VERSION; +} /* timeout_version() */ + + +TIMEOUT_PUBLIC const char *timeout_vendor(void) { + return TIMEOUT_VENDOR; +} /* timeout_version() */ + + +TIMEOUT_PUBLIC int timeout_v_rel(void) { + return TIMEOUT_V_REL; +} /* timeout_version() */ + + +TIMEOUT_PUBLIC int timeout_v_abi(void) { + return TIMEOUT_V_ABI; +} /* timeout_version() */ + + +TIMEOUT_PUBLIC int timeout_v_api(void) { + return TIMEOUT_V_API; +} /* timeout_version() */ + diff --git a/src/ext/timeouts/timeout.h b/src/ext/timeouts/timeout.h new file mode 100644 index 0000000000..b35874e153 --- /dev/null +++ b/src/ext/timeouts/timeout.h @@ -0,0 +1,256 @@ +/* ========================================================================== + * timeout.h - Tickless hierarchical timing wheel. + * -------------------------------------------------------------------------- + * Copyright (c) 2013, 2014 William Ahern + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the + * following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * ========================================================================== + */ +#ifndef TIMEOUT_H +#define TIMEOUT_H + +#include <stdbool.h> /* bool */ +#include <stdio.h> /* FILE */ + +#include <inttypes.h> /* PRIu64 PRIx64 PRIX64 uint64_t */ + +#include "tor_queue.h" /* TAILQ(3) */ + + +/* + * V E R S I O N I N T E R F A C E S + * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#if !defined TIMEOUT_PUBLIC +#define TIMEOUT_PUBLIC +#endif + +#define TIMEOUT_VERSION TIMEOUT_V_REL +#define TIMEOUT_VENDOR "william@25thandClement.com" + +#define TIMEOUT_V_REL 0x20160226 +#define TIMEOUT_V_ABI 0x20160224 +#define TIMEOUT_V_API 0x20160226 + +TIMEOUT_PUBLIC int timeout_version(void); + +TIMEOUT_PUBLIC const char *timeout_vendor(void); + +TIMEOUT_PUBLIC int timeout_v_rel(void); + +TIMEOUT_PUBLIC int timeout_v_abi(void); + +TIMEOUT_PUBLIC int timeout_v_api(void); + + +/* + * I N T E G E R T Y P E I N T E R F A C E S + * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#define TIMEOUT_C(n) UINT64_C(n) +#define TIMEOUT_PRIu PRIu64 +#define TIMEOUT_PRIx PRIx64 +#define TIMEOUT_PRIX PRIX64 + +#define TIMEOUT_mHZ TIMEOUT_C(1000) +#define TIMEOUT_uHZ TIMEOUT_C(1000000) +#define TIMEOUT_nHZ TIMEOUT_C(1000000000) + +typedef uint64_t timeout_t; + +#define timeout_error_t int /* for documentation purposes */ + + +/* + * C A L L B A C K I N T E R F A C E + * + * Callback function parameters unspecified to make embedding into existing + * applications easier. + * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef TIMEOUT_CB_OVERRIDE +struct timeout_cb { + void (*fn)(void); + void *arg; +}; /* struct timeout_cb */ +#endif + +/* + * T I M E O U T I N T E R F A C E S + * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef TIMEOUT_DISABLE_INTERVALS +#define TIMEOUT_INT 0x01 /* interval (repeating) timeout */ +#endif +#define TIMEOUT_ABS 0x02 /* treat timeout values as absolute */ + +#define TIMEOUT_INITIALIZER(flags) { (flags) } + +#define timeout_setcb(to, fn, arg) do { \ + (to)->callback.fn = (fn); \ + (to)->callback.arg = (arg); \ +} while (0) + +struct timeout { + int flags; + + timeout_t expires; + /* absolute expiration time */ + + struct timeout_list *pending; + /* timeout list if pending on wheel or expiry queue */ + + TOR_TAILQ_ENTRY(timeout) tqe; + /* entry member for struct timeout_list lists */ + +#ifndef TIMEOUT_DISABLE_CALLBACKS + struct timeout_cb callback; + /* optional callback information */ +#endif + +#ifndef TIMEOUT_DISABLE_INTERVALS + timeout_t interval; + /* timeout interval if periodic */ +#endif + +#ifndef TIMEOUT_DISABLE_RELATIVE_ACCESS + struct timeouts *timeouts; + /* timeouts collection if member of */ +#endif +}; /* struct timeout */ + + +TIMEOUT_PUBLIC struct timeout *timeout_init(struct timeout *, int); +/* initialize timeout structure (same as TIMEOUT_INITIALIZER) */ + +#ifndef TIMEOUT_DISABLE_RELATIVE_ACCESS +TIMEOUT_PUBLIC bool timeout_pending(struct timeout *); +/* true if on timing wheel, false otherwise */ + +TIMEOUT_PUBLIC bool timeout_expired(struct timeout *); +/* true if on expired queue, false otherwise */ + +TIMEOUT_PUBLIC void timeout_del(struct timeout *); +/* remove timeout from any timing wheel (okay if not member of any) */ +#endif + +/* + * T I M I N G W H E E L I N T E R F A C E S + * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +struct timeouts; + +TIMEOUT_PUBLIC struct timeouts *timeouts_open(timeout_t, timeout_error_t *); +/* open a new timing wheel, setting optional HZ (for float conversions) */ + +TIMEOUT_PUBLIC void timeouts_close(struct timeouts *); +/* destroy timing wheel */ + +TIMEOUT_PUBLIC timeout_t timeouts_hz(struct timeouts *); +/* return HZ setting (for float conversions) */ + +TIMEOUT_PUBLIC void timeouts_update(struct timeouts *, timeout_t); +/* update timing wheel with current absolute time */ + +TIMEOUT_PUBLIC void timeouts_step(struct timeouts *, timeout_t); +/* step timing wheel by relative time */ + +TIMEOUT_PUBLIC timeout_t timeouts_get_curtime(struct timeouts *); +/* Return the current tick. */ + +TIMEOUT_PUBLIC timeout_t timeouts_timeout(struct timeouts *); +/* return interval to next required update */ + +TIMEOUT_PUBLIC void timeouts_add(struct timeouts *, struct timeout *, timeout_t); +/* add timeout to timing wheel */ + +TIMEOUT_PUBLIC void timeouts_del(struct timeouts *, struct timeout *); +/* remove timeout from any timing wheel or expired queue (okay if on neither) */ + +TIMEOUT_PUBLIC struct timeout *timeouts_get(struct timeouts *); +/* return any expired timeout (caller should loop until NULL-return) */ + +TIMEOUT_PUBLIC bool timeouts_pending(struct timeouts *); +/* return true if any timeouts pending on timing wheel */ + +TIMEOUT_PUBLIC bool timeouts_expired(struct timeouts *); +/* return true if any timeouts on expired queue */ + +TIMEOUT_PUBLIC bool timeouts_check(struct timeouts *, FILE *); +/* return true if invariants hold. describes failures to optional file handle. */ + +#define TIMEOUTS_PENDING 0x10 +#define TIMEOUTS_EXPIRED 0x20 +#define TIMEOUTS_ALL (TIMEOUTS_PENDING|TIMEOUTS_EXPIRED) +#define TIMEOUTS_CLEAR 0x40 + +#define TIMEOUTS_IT_INITIALIZER(flags) { (flags), 0, 0, 0, 0 } + +#define TIMEOUTS_IT_INIT(cur, _flags) do { \ + (cur)->flags = (_flags); \ + (cur)->pc = 0; \ +} while (0) + +struct timeouts_it { + int flags; + unsigned pc, i, j; + struct timeout *to; +}; /* struct timeouts_it */ + +TIMEOUT_PUBLIC struct timeout *timeouts_next(struct timeouts *, struct timeouts_it *); +/* return next timeout in pending wheel or expired queue. caller can delete + * the returned timeout, but should not otherwise manipulate the timing + * wheel. in particular, caller SHOULD NOT delete any other timeout as that + * could invalidate cursor state and trigger a use-after-free. + */ + +#define TIMEOUTS_FOREACH(var, T, flags) \ + struct timeouts_it _it = TIMEOUTS_IT_INITIALIZER((flags)); \ + while (((var) = timeouts_next((T), &_it))) + + +/* + * B O N U S W H E E L I N T E R F A C E S + * + * I usually use floating point timeouts in all my code, but it's cleaner to + * separate it to keep the core algorithmic code simple. + * + * Using macros instead of static inline routines where <math.h> routines + * might be used to keep -lm linking optional. + * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include <math.h> /* ceil(3) */ + +#define timeouts_f2i(T, f) \ + ((timeout_t)ceil((f) * timeouts_hz((T)))) /* prefer late expiration over early */ + +#define timeouts_i2f(T, i) \ + ((double)(i) / timeouts_hz((T))) + +#define timeouts_addf(T, to, timeout) \ + timeouts_add((T), (to), timeouts_f2i((T), (timeout))) + +#endif /* TIMEOUT_H */ diff --git a/src/ext/tinytest.c b/src/ext/tinytest.c index f6baeeb9a5..3fb1b39c71 100644 --- a/src/ext/tinytest.c +++ b/src/ext/tinytest.c @@ -69,15 +69,16 @@ static int n_skipped = 0; /**< Number of tests that have been skipped. */ static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/ static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */ static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */ -const char *verbosity_flag = ""; +static const char *verbosity_flag = ""; -const struct testlist_alias_t *cfg_aliases=NULL; +static const struct testlist_alias_t *cfg_aliases=NULL; enum outcome { SKIP=2, OK=1, FAIL=0 }; static enum outcome cur_test_outcome = 0; -const char *cur_test_prefix = NULL; /**< prefix of the current test group */ +/** prefix of the current test group */ +static const char *cur_test_prefix = NULL; /** Name of the current test, if we haven't logged is yet. Used for --quiet */ -const char *cur_test_name = NULL; +static const char *cur_test_name = NULL; #ifdef _WIN32 /* Copy of argv[0] for win32. */ diff --git a/src/or/buffers.c b/src/or/buffers.c index f93cc48f33..8b9a53c699 100644 --- a/src/or/buffers.c +++ b/src/or/buffers.c @@ -107,7 +107,7 @@ chunk_repack(chunk_t *chunk) /** Keep track of total size of allocated chunks for consistency asserts */ static size_t total_bytes_allocated_in_chunks = 0; static void -chunk_free_unchecked(chunk_t *chunk) +buf_chunk_free_unchecked(chunk_t *chunk) { if (!chunk) return; @@ -228,7 +228,7 @@ buf_pullup(buf_t *buf, size_t bytes) dest->next = src->next; if (buf->tail == src) buf->tail = dest; - chunk_free_unchecked(src); + buf_chunk_free_unchecked(src); } else { memcpy(CHUNK_WRITE_PTR(dest), src->data, n); dest->datalen += n; @@ -274,7 +274,7 @@ buf_remove_from_front(buf_t *buf, size_t n) buf->head = victim->next; if (buf->tail == victim) buf->tail = NULL; - chunk_free_unchecked(victim); + buf_chunk_free_unchecked(victim); } } check(); @@ -314,7 +314,7 @@ buf_clear(buf_t *buf) buf->datalen = 0; for (chunk = buf->head; chunk; chunk = next) { next = chunk->next; - chunk_free_unchecked(chunk); + buf_chunk_free_unchecked(chunk); } buf->head = buf->tail = NULL; } @@ -509,12 +509,12 @@ read_to_chunk_tls(buf_t *buf, chunk_t *chunk, tor_tls_t *tls, * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on * error; else return the number of bytes read. */ -/* XXXX024 indicate "read blocked" somehow? */ +/* XXXX indicate "read blocked" somehow? */ int read_to_buf(tor_socket_t s, size_t at_most, buf_t *buf, int *reached_eof, int *socket_error) { - /* XXXX024 It's stupid to overload the return values for these functions: + /* XXXX It's stupid to overload the return values for these functions: * "error status" and "number of bytes read" are not mutually exclusive. */ int r = 0; @@ -687,7 +687,7 @@ flush_chunk_tls(tor_tls_t *tls, buf_t *buf, chunk_t *chunk, int flush_buf(tor_socket_t s, buf_t *buf, size_t sz, size_t *buf_flushlen) { - /* XXXX024 It's stupid to overload the return values for these functions: + /* XXXX It's stupid to overload the return values for these functions: * "error status" and "number of bytes flushed" are not mutually exclusive. */ int r; diff --git a/src/or/channel.c b/src/or/channel.c index 5f69a0864b..87fa721089 100644 --- a/src/or/channel.c +++ b/src/or/channel.c @@ -122,7 +122,7 @@ STATIC uint64_t estimated_total_queue_size = 0; * If more than one channel exists, follow the next_with_same_id pointer * as a linked list. */ -HT_HEAD(channel_idmap, channel_idmap_entry_s) channel_identity_map = +static HT_HEAD(channel_idmap, channel_idmap_entry_s) channel_identity_map = HT_INITIALIZER(); typedef struct channel_idmap_entry_s { @@ -145,9 +145,9 @@ channel_idmap_eq(const channel_idmap_entry_t *a, } HT_PROTOTYPE(channel_idmap, channel_idmap_entry_s, node, channel_idmap_hash, - channel_idmap_eq); + channel_idmap_eq) HT_GENERATE2(channel_idmap, channel_idmap_entry_s, node, channel_idmap_hash, - channel_idmap_eq, 0.5, tor_reallocarray_, tor_free_); + channel_idmap_eq, 0.5, tor_reallocarray_, tor_free_) static cell_queue_entry_t * cell_queue_entry_dup(cell_queue_entry_t *q); #if 0 @@ -3510,7 +3510,7 @@ channel_dump_statistics, (channel_t *chan, int severity)) have_remote_addr = channel_get_addr_if_possible(chan, &remote_addr); if (have_remote_addr) { char *actual = tor_strdup(channel_get_actual_remote_descr(chan)); - remote_addr_str = tor_dup_addr(&remote_addr); + remote_addr_str = tor_addr_to_str_dup(&remote_addr); tor_log(severity, LD_GENERAL, " * Channel " U64_FORMAT " says its remote address" " is %s, and gives a canonical description of \"%s\" and an " @@ -4524,8 +4524,8 @@ channel_update_xmit_queue_size(channel_t *chan) /* Next, adjust by the overhead factor, if any is available */ if (chan->get_overhead_estimate) { overhead = chan->get_overhead_estimate(chan); - if (overhead >= 1.0f) { - queued *= overhead; + if (overhead >= 1.0) { + queued = (uint64_t)(queued * overhead); } else { /* Ignore silly overhead factors */ log_notice(LD_CHANNEL, "Ignoring silly overhead factor %f", overhead); diff --git a/src/or/channel.h b/src/or/channel.h index 129c0c2013..78e1b71014 100644 --- a/src/or/channel.h +++ b/src/or/channel.h @@ -18,7 +18,7 @@ typedef void (*channel_cell_handler_fn_ptr)(channel_t *, cell_t *); typedef void (*channel_var_cell_handler_fn_ptr)(channel_t *, var_cell_t *); struct cell_queue_entry_s; -TOR_SIMPLEQ_HEAD(chan_cell_queue, cell_queue_entry_s) incoming_queue; +TOR_SIMPLEQ_HEAD(chan_cell_queue, cell_queue_entry_s); typedef struct chan_cell_queue chan_cell_queue_t; /** @@ -469,6 +469,10 @@ void channel_notify_flushed(channel_t *chan); /* Handle stuff we need to do on open like notifying circuits */ void channel_do_open_actions(channel_t *chan); +#ifdef TOR_UNIT_TESTS +extern uint64_t estimated_total_queue_size; +#endif + #endif /* Helper functions to perform operations on channels */ diff --git a/src/or/channeltls.c b/src/or/channeltls.c index c65af5d040..2bb88dd505 100644 --- a/src/or/channeltls.c +++ b/src/or/channeltls.c @@ -22,6 +22,7 @@ #include "channeltls.h" #include "circuitmux.h" #include "circuitmux_ewma.h" +#include "command.h" #include "config.h" #include "connection.h" #include "connection_or.h" @@ -51,7 +52,7 @@ uint64_t stats_n_authenticate_cells_processed = 0; uint64_t stats_n_authorize_cells_processed = 0; /** Active listener, if any */ -channel_listener_t *channel_tls_listener = NULL; +static channel_listener_t *channel_tls_listener = NULL; /* channel_tls_t method declarations */ @@ -445,7 +446,7 @@ channel_tls_free_method(channel_t *chan) static double channel_tls_get_overhead_estimate_method(channel_t *chan) { - double overhead = 1.0f; + double overhead = 1.0; channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan); tor_assert(tlschan); @@ -462,7 +463,8 @@ channel_tls_get_overhead_estimate_method(channel_t *chan) * Never estimate more than 2.0; otherwise we get silly large estimates * at the very start of a new TLS connection. */ - if (overhead > 2.0f) overhead = 2.0f; + if (overhead > 2.0) + overhead = 2.0; } log_debug(LD_CHANNEL, @@ -554,7 +556,7 @@ channel_tls_get_remote_descr_method(channel_t *chan, int flags) break; case GRD_FLAG_ORIGINAL: /* Actual address with port */ - addr_str = tor_dup_addr(&(tlschan->conn->real_addr)); + addr_str = tor_addr_to_str_dup(&(tlschan->conn->real_addr)); tor_snprintf(buf, MAX_DESCR_LEN + 1, "%s:%u", addr_str, conn->port); tor_free(addr_str); @@ -567,7 +569,7 @@ channel_tls_get_remote_descr_method(channel_t *chan, int flags) break; case GRD_FLAG_ORIGINAL|GRD_FLAG_ADDR_ONLY: /* Actual address, no port */ - addr_str = tor_dup_addr(&(tlschan->conn->real_addr)); + addr_str = tor_addr_to_str_dup(&(tlschan->conn->real_addr)); strlcpy(buf, addr_str, sizeof(buf)); tor_free(addr_str); answer = buf; @@ -797,6 +799,7 @@ static int channel_tls_write_packed_cell_method(channel_t *chan, packed_cell_t *packed_cell) { + tor_assert(chan); channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan); size_t cell_network_size = get_cell_network_size(chan->wide_circ_ids); int written = 0; diff --git a/src/or/channeltls.h b/src/or/channeltls.h index a4d9c7a095..8b5863a461 100644 --- a/src/or/channeltls.h +++ b/src/or/channeltls.h @@ -52,6 +52,14 @@ void channel_tls_update_marks(or_connection_t *conn); /* Cleanup at shutdown */ void channel_tls_free_all(void); +extern uint64_t stats_n_authorize_cells_processed; +extern uint64_t stats_n_authenticate_cells_processed; +extern uint64_t stats_n_versions_cells_processed; +extern uint64_t stats_n_netinfo_cells_processed; +extern uint64_t stats_n_vpadding_cells_processed; +extern uint64_t stats_n_certs_cells_processed; +extern uint64_t stats_n_auth_challenge_cells_processed; + #ifdef CHANNELTLS_PRIVATE STATIC void channel_tls_process_certs_cell(var_cell_t *cell, channel_tls_t *tlschan); diff --git a/src/or/circpathbias.c b/src/or/circpathbias.c index 552947eba2..9f93e737f7 100644 --- a/src/or/circpathbias.c +++ b/src/or/circpathbias.c @@ -85,7 +85,6 @@ pathbias_get_notice_rate(const or_options_t *options) DFLT_PATH_BIAS_NOTICE_PCT, 0, 100)/100.0; } -/* XXXX024 I'd like to have this be static again, but entrynodes.c needs it. */ /** The circuit success rate below which we issue a warn */ static double pathbias_get_warn_rate(const or_options_t *options) @@ -98,7 +97,7 @@ pathbias_get_warn_rate(const or_options_t *options) DFLT_PATH_BIAS_WARN_PCT, 0, 100)/100.0; } -/* XXXX024 I'd like to have this be static again, but entrynodes.c needs it. */ +/* XXXX I'd like to have this be static again, but entrynodes.c needs it. */ /** * The extreme rate is the rate at which we would drop the guard, * if pb_dropguard is also set. Otherwise we just warn. @@ -114,7 +113,7 @@ pathbias_get_extreme_rate(const or_options_t *options) DFLT_PATH_BIAS_EXTREME_PCT, 0, 100)/100.0; } -/* XXXX024 I'd like to have this be static again, but entrynodes.c needs it. */ +/* XXXX I'd like to have this be static again, but entrynodes.c needs it. */ /** * If 1, we actually disable use of guards that fall below * the extreme_pct. diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index 820724adea..13cc16670c 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -47,10 +47,6 @@ #include "routerset.h" #include "crypto.h" -#ifndef MIN -#define MIN(a,b) ((a)<(b)?(a):(b)) -#endif - static channel_t * channel_connect_for_circuit(const tor_addr_t *addr, uint16_t port, const char *id_digest); @@ -809,6 +805,7 @@ circuit_pick_create_handshake(uint8_t *cell_type_out, uint16_t *handshake_type_out, const extend_info_t *ei) { + /* XXXX029 Remove support for deciding to use TAP. */ if (!tor_mem_is_zero((const char*)ei->curve25519_onion_key.public_key, CURVE25519_PUBKEY_LEN) && circuits_can_use_ntor()) { @@ -835,9 +832,8 @@ circuit_pick_extend_handshake(uint8_t *cell_type_out, { uint8_t t; circuit_pick_create_handshake(&t, handshake_type_out, ei); - /* XXXX024 The check for whether the node has a curve25519 key is a bad - * proxy for whether it can do extend2 cells; once a version that - * handles extend2 cells is out, remove it. */ + + /* XXXX029 Remove support for deciding to use TAP. */ if (node_prev && *handshake_type_out != ONION_HANDSHAKE_TYPE_TAP && (node_has_curve25519_onion_key(node_prev) || @@ -888,14 +884,12 @@ circuit_send_next_onion_skin(origin_circuit_t *circ) */ circuit_pick_create_handshake(&cc.cell_type, &cc.handshake_type, circ->cpath->extend_info); - note_request("cell: create", 1); } else { /* We are not an OR, and we're building the first hop of a circuit to a * new OR: we can be speedy and use CREATE_FAST to save an RSA operation * and a DH operation. */ cc.cell_type = CELL_CREATE_FAST; cc.handshake_type = ONION_HANDSHAKE_TYPE_FAST; - note_request("cell: create fast", 1); } len = onion_skin_create(cc.handshake_type, @@ -1028,7 +1022,6 @@ circuit_send_next_onion_skin(origin_circuit_t *circ) ec.create_cell.handshake_len = len; log_info(LD_CIRC,"Sending extend relay cell."); - note_request("cell: extend", 1); { uint8_t command = 0; uint16_t payload_len=0; @@ -2146,7 +2139,6 @@ choose_good_middle_server(uint8_t purpose, * If <b>state</b> is NULL, we're choosing a router to serve as an entry * guard, not for any particular circuit. */ -/* XXXX024 I'd like to have this be static again, but entrynodes.c needs it. */ const node_t * choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state) { @@ -2179,7 +2171,7 @@ choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state) * This is an incomplete fix, but is no worse than the previous behaviour, * and only applies to minimal, testing tor networks * (so it's no less secure) */ - /*XXXX025 use the using_as_guard flag to accomplish this.*/ + /*XXXX++ use the using_as_guard flag to accomplish this.*/ if (options->UseEntryGuards && (!options->TestingTorNetwork || smartlist_len(nodelist_get_list()) > smartlist_len(get_entry_guards()) diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 1efb7ef4d0..d2ba7d4781 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -109,7 +109,7 @@ HT_GENERATE2(chan_circid_map, chan_circid_circuit_map_t, node, * used to improve performance when many cells arrive in a row from the * same circuit. */ -chan_circid_circuit_map_t *_last_circid_chan_ent = NULL; +static chan_circid_circuit_map_t *_last_circid_chan_ent = NULL; /** Implementation helper for circuit_set_{p,n}_circid_channel: A circuit ID * and/or channel for circ has just changed from <b>old_chan, old_id</b> diff --git a/src/or/circuitmux.c b/src/or/circuitmux.c index cc1c4cd401..038904e68a 100644 --- a/src/or/circuitmux.c +++ b/src/or/circuitmux.c @@ -362,7 +362,7 @@ HT_HEAD(chanid_circid_muxinfo_map, chanid_circid_muxinfo_t); /* Emit a bunch of hash table stuff */ HT_PROTOTYPE(chanid_circid_muxinfo_map, chanid_circid_muxinfo_t, node, - chanid_circid_entry_hash, chanid_circid_entries_eq); + chanid_circid_entry_hash, chanid_circid_entries_eq) HT_GENERATE2(chanid_circid_muxinfo_map, chanid_circid_muxinfo_t, node, chanid_circid_entry_hash, chanid_circid_entries_eq, 0.6, tor_reallocarray_, tor_free_) diff --git a/src/or/circuitmux_ewma.h b/src/or/circuitmux_ewma.h index 58aac1e196..a7b8961ac6 100644 --- a/src/or/circuitmux_ewma.h +++ b/src/or/circuitmux_ewma.h @@ -12,13 +12,8 @@ #include "or.h" #include "circuitmux.h" -/* Everything but circuitmux_ewma.c should see this extern */ -#ifndef TOR_CIRCUITMUX_EWMA_C_ - extern circuitmux_policy_t ewma_policy; -#endif /* !(TOR_CIRCUITMUX_EWMA_C_) */ - /* Externally visible EWMA functions */ int cell_ewma_enabled(void); unsigned int cell_ewma_get_tick(void); diff --git a/src/or/circuituse.c b/src/or/circuituse.c index 2c724dee05..f344703331 100644 --- a/src/or/circuituse.c +++ b/src/or/circuituse.c @@ -203,7 +203,7 @@ circuit_is_better(const origin_circuit_t *oa, const origin_circuit_t *ob, timercmp(&a->timestamp_began, &b->timestamp_began, OP_GT)) return 1; if (ob->build_state->is_internal) - /* XXX023 what the heck is this internal thing doing here. I + /* XXXX++ what the heck is this internal thing doing here. I * think we can get rid of it. circuit_is_acceptable() already * makes sure that is_internal is exactly what we need it to * be. -RD */ @@ -222,7 +222,7 @@ circuit_is_better(const origin_circuit_t *oa, const origin_circuit_t *ob, break; } - /* XXXX023 Maybe this check should get a higher priority to avoid + /* XXXX Maybe this check should get a higher priority to avoid * using up circuits too rapidly. */ a_bits = connection_edge_update_circuit_isolation(conn, @@ -1067,7 +1067,7 @@ circuit_predict_and_launch_new(void) if (rep_hist_get_predicted_internal(now, &hidserv_needs_uptime, &hidserv_needs_capacity) && ((num_uptime_internal<2 && hidserv_needs_uptime) || - num_internal<2) + num_internal<3) && router_have_consensus_path() != CONSENSUS_PATH_UNKNOWN) { if (hidserv_needs_uptime) flags |= CIRCLAUNCH_NEED_UPTIME; @@ -1936,8 +1936,8 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn, return -1; } } else { - /* XXXX024 Duplicates checks in connection_ap_handshake_attach_circuit: - * refactor into a single function? */ + /* XXXX Duplicates checks in connection_ap_handshake_attach_circuit: + * refactor into a single function. */ const node_t *node = node_get_by_nickname(conn->chosen_exit_name, 1); int opt = conn->chosen_exit_optional; if (node && !connection_ap_can_use_exit(conn, node)) { @@ -2028,7 +2028,8 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn, char *hexdigest = conn->chosen_exit_name+1; tor_addr_t addr; if (strlen(hexdigest) < HEX_DIGEST_LEN || - base16_decode(digest,DIGEST_LEN,hexdigest,HEX_DIGEST_LEN)<0) { + base16_decode(digest,DIGEST_LEN, + hexdigest,HEX_DIGEST_LEN) != DIGEST_LEN) { log_info(LD_DIR, "Broken exit digest on tunnel conn. Closing."); return -1; } @@ -2146,10 +2147,11 @@ optimistic_data_enabled(void) { const or_options_t *options = get_options(); if (options->OptimisticData < 0) { - /* XXX023 consider having auto default to 1 rather than 0 before - * the 0.2.3 branch goes stable. See bug 3617. -RD */ + /* Note: this default was 0 before #18815 was merged. We can't take the + * parameter out of the consensus until versions before that are all + * obsolete. */ const int32_t enabled = - networkstatus_get_param(NULL, "UseOptimisticData", 0, 0, 1); + networkstatus_get_param(NULL, "UseOptimisticData", /*default*/ 1, 0, 1); return (int)enabled; } return options->OptimisticData; @@ -2415,7 +2417,7 @@ connection_ap_handshake_attach_circuit(entry_connection_t *conn) /* find the circuit that we should use, if there is one. */ retval = circuit_get_open_circ_or_launch( conn, CIRCUIT_PURPOSE_C_GENERAL, &circ); - if (retval < 1) // XXX023 if we totally fail, this still returns 0 -RD + if (retval < 1) // XXXX++ if we totally fail, this still returns 0 -RD return retval; log_debug(LD_APP|LD_CIRC, @@ -2590,7 +2592,7 @@ mark_circuit_unusable_for_new_conns(origin_circuit_t *circ) const or_options_t *options = get_options(); tor_assert(circ); - /* XXXX025 This is a kludge; we're only keeping it around in case there's + /* XXXX This is a kludge; we're only keeping it around in case there's * something that doesn't check unusable_for_new_conns, and to avoid * deeper refactoring of our expiration logic. */ if (! circ->base_.timestamp_dirty) diff --git a/src/or/config.c b/src/or/config.c index 908f098ce7..a677f21955 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -65,9 +65,6 @@ #include <systemd/sd-daemon.h> #endif -/* From main.c */ -extern int quiet_level; - /* Prefix used to indicate a Unix socket in a FooPort configuration. */ static const char unix_socket_prefix[] = "unix:"; @@ -99,7 +96,7 @@ static config_abbrev_t option_abbrevs_[] = { { "BandwidthRateBytes", "BandwidthRate", 0, 0}, { "BandwidthBurstBytes", "BandwidthBurst", 0, 0}, { "DirFetchPostPeriod", "StatusFetchPeriod", 0, 0}, - { "DirServer", "DirAuthority", 0, 0}, /* XXXX024 later, make this warn? */ + { "DirServer", "DirAuthority", 0, 0}, /* XXXX later, make this warn? */ { "MaxConn", "ConnLimit", 0, 1}, { "MaxMemInCellQueues", "MaxMemInQueues", 0, 0}, { "ORBindAddress", "ORListenAddress", 0, 0}, @@ -328,6 +325,7 @@ static config_var_t option_vars_[] = { VAR("MaxMemInQueues", MEMUNIT, MaxMemInQueues_raw, "0"), OBSOLETE("MaxOnionsPending"), V(MaxOnionQueueDelay, MSEC_INTERVAL, "1750 msec"), + V(MaxUnparseableDescSizeToLog, MEMUNIT, "10 MB"), V(MinMeasuredBWsForAuthToIgnoreAdvertised, INT, "500"), V(MyFamily, STRING, NULL), V(NewCircuitPeriod, INTERVAL, "30 seconds"), @@ -442,6 +440,7 @@ static config_var_t option_vars_[] = { V(UseNTorHandshake, AUTOBOOL, "1"), V(User, STRING, NULL), V(UserspaceIOCPBuffers, BOOL, "0"), + V(AuthDirSharedRandomness, BOOL, "1"), OBSOLETE("V1AuthoritativeDirectory"), OBSOLETE("V2AuthoritativeDirectory"), VAR("V3AuthoritativeDirectory",BOOL, V3AuthoritativeDir, "0"), @@ -1999,11 +1998,6 @@ static const struct { { "--list-fingerprint", TAKES_NO_ARGUMENT }, { "--keygen", TAKES_NO_ARGUMENT }, { "--newpass", TAKES_NO_ARGUMENT }, -#if 0 -/* XXXX028: This is not working yet in 0.2.7, so disabling with the - * minimal code modification. */ - { "--master-key", ARGUMENT_NECESSARY }, -#endif { "--no-passphrase", TAKES_NO_ARGUMENT }, { "--passphrase-fd", ARGUMENT_NECESSARY }, { "--verify-config", TAKES_NO_ARGUMENT }, @@ -2077,7 +2071,7 @@ config_parse_commandline(int argc, char **argv, int ignore_errors, if (want_arg == ARGUMENT_NECESSARY && is_last) { if (ignore_errors) { - arg = strdup(""); + arg = tor_strdup(""); } else { log_warn(LD_CONFIG,"Command-line option '%s' with no value. Failing.", argv[i]); @@ -2484,7 +2478,6 @@ is_local_addr, (const tor_addr_t *addr)) if (get_options()->EnforceDistinctSubnets == 0) return 0; if (tor_addr_family(addr) == AF_INET) { - /*XXXX023 IP6 what corresponds to an /24? */ uint32_t ip = tor_addr_to_ipv4h(addr); /* It's possible that this next check will hit before the first time @@ -2678,7 +2671,7 @@ options_validate_cb(void *old_options, void *options, void *default_options, #define REJECT(arg) \ STMT_BEGIN *msg = tor_strdup(arg); return -1; STMT_END -#ifdef __GNUC__ +#if defined(__GNUC__) && __GNUC__ <= 3 #define COMPLAIN(args...) \ STMT_BEGIN log_warn(LD_CONFIG, args); STMT_END #else @@ -2791,7 +2784,8 @@ options_validate(or_options_t *old_options, or_options_t *options, } else { if (!is_legal_nickname(options->Nickname)) { tor_asprintf(msg, - "Nickname '%s' is wrong length or contains illegal characters.", + "Nickname '%s', nicknames must be between 1 and 19 characters " + "inclusive, and must contain only the characters [a-zA-Z0-9].", options->Nickname); return -1; } @@ -4127,11 +4121,11 @@ have_enough_mem_for_dircache(const or_options_t *options, size_t total_mem, if (options->DirCache) { if (total_mem < DIRCACHE_MIN_BANDWIDTH) { if (options->BridgeRelay) { - *msg = strdup("Running a Bridge with less than " + *msg = tor_strdup("Running a Bridge with less than " STRINGIFY(DIRCACHE_MIN_MB_BANDWIDTH) " MB of memory is " "not recommended."); } else { - *msg = strdup("Being a directory cache (default) with less than " + *msg = tor_strdup("Being a directory cache (default) with less than " STRINGIFY(DIRCACHE_MIN_MB_BANDWIDTH) " MB of memory is " "not recommended and may consume most of the available " "resources, consider disabling this functionality by " @@ -4140,7 +4134,7 @@ have_enough_mem_for_dircache(const or_options_t *options, size_t total_mem, } } else { if (total_mem >= DIRCACHE_MIN_BANDWIDTH) { - *msg = strdup("DirCache is disabled and we are configured as a " + *msg = tor_strdup("DirCache is disabled and we are configured as a " "relay. This may disqualify us from becoming a guard in the " "future."); } @@ -5025,7 +5019,7 @@ config_register_addressmaps(const or_options_t *options) /** As addressmap_register(), but detect the wildcarded status of "from" and * "to", and do not steal a reference to <b>to</b>. */ -/* XXXX024 move to connection_edge.c */ +/* XXXX move to connection_edge.c */ int addressmap_register_auto(const char *from, const char *to, time_t expires, @@ -5333,7 +5327,7 @@ parse_bridge_line(const char *line) goto err; } if (base16_decode(bridge_line->digest, DIGEST_LEN, - fingerprint, HEX_DIGEST_LEN)<0) { + fingerprint, HEX_DIGEST_LEN) != DIGEST_LEN) { log_warn(LD_CONFIG, "Unable to decode Bridge key digest."); goto err; } @@ -5776,7 +5770,7 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, } else if (!strcmpstart(flag, "weight=")) { int ok; const char *wstring = flag + strlen("weight="); - weight = tor_parse_double(wstring, 0, UINT64_MAX, &ok, NULL); + weight = tor_parse_double(wstring, 0, (double)UINT64_MAX, &ok, NULL); if (!ok) { log_warn(LD_CONFIG, "Invalid weight '%s' on DirAuthority line.",flag); weight=1.0; @@ -5784,7 +5778,8 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, } else if (!strcasecmpstart(flag, "v3ident=")) { char *idstr = flag + strlen("v3ident="); if (strlen(idstr) != HEX_DIGEST_LEN || - base16_decode(v3_digest, DIGEST_LEN, idstr, HEX_DIGEST_LEN)<0) { + base16_decode(v3_digest, DIGEST_LEN, + idstr, HEX_DIGEST_LEN) != DIGEST_LEN) { log_warn(LD_CONFIG, "Bad v3 identity digest '%s' on DirAuthority line", flag); } else { @@ -5833,7 +5828,8 @@ parse_dir_authority_line(const char *line, dirinfo_type_t required_type, fingerprint, (int)strlen(fingerprint)); goto err; } - if (base16_decode(digest, DIGEST_LEN, fingerprint, HEX_DIGEST_LEN)<0) { + if (base16_decode(digest, DIGEST_LEN, + fingerprint, HEX_DIGEST_LEN) != DIGEST_LEN) { log_warn(LD_CONFIG, "Unable to decode DirAuthority key digest."); goto err; } @@ -5901,8 +5897,8 @@ parse_dir_fallback_line(const char *line, orport = (int)tor_parse_long(cp+strlen("orport="), 10, 1, 65535, &ok, NULL); } else if (!strcmpstart(cp, "id=")) { - ok = !base16_decode(id, DIGEST_LEN, - cp+strlen("id="), strlen(cp)-strlen("id=")); + ok = base16_decode(id, DIGEST_LEN, cp+strlen("id="), + strlen(cp)-strlen("id=")) == DIGEST_LEN; } else if (!strcasecmpstart(cp, "ipv6=")) { if (ipv6_addrport_ptr) { log_warn(LD_CONFIG, "Redundant ipv6 addr/port on FallbackDir line"); @@ -5920,7 +5916,7 @@ parse_dir_fallback_line(const char *line, } else if (!strcmpstart(cp, "weight=")) { int ok; const char *wstring = cp + strlen("weight="); - weight = tor_parse_double(wstring, 0, UINT64_MAX, &ok, NULL); + weight = tor_parse_double(wstring, 0, (double)UINT64_MAX, &ok, NULL); if (!ok) { log_warn(LD_CONFIG, "Invalid weight '%s' on FallbackDir line.", cp); weight=1.0; @@ -7229,10 +7225,10 @@ init_libevent(const or_options_t *options) * * Note: Consider using the get_datadir_fname* macros in or.h. */ -char * -options_get_datadir_fname2_suffix(const or_options_t *options, - const char *sub1, const char *sub2, - const char *suffix) +MOCK_IMPL(char *, +options_get_datadir_fname2_suffix,(const or_options_t *options, + const char *sub1, const char *sub2, + const char *suffix)) { char *fname = NULL; size_t len; @@ -7566,7 +7562,7 @@ static void config_maybe_load_geoip_files_(const or_options_t *options, const or_options_t *old_options) { - /* XXXX024 Reload GeoIPFile on SIGHUP. -NM */ + /* XXXX Reload GeoIPFile on SIGHUP. -NM */ if (options->GeoIPFile && ((!old_options || !opt_streq(old_options->GeoIPFile, diff --git a/src/or/config.h b/src/or/config.h index 02121cf95c..a0fe6e4805 100644 --- a/src/or/config.h +++ b/src/or/config.h @@ -53,9 +53,11 @@ config_line_t *option_get_assignment(const or_options_t *options, const char *key); int options_save_current(void); const char *get_torrc_fname(int defaults_fname); -char *options_get_datadir_fname2_suffix(const or_options_t *options, - const char *sub1, const char *sub2, - const char *suffix); +MOCK_DECL(char *, + options_get_datadir_fname2_suffix, + (const or_options_t *options, + const char *sub1, const char *sub2, + const char *suffix)); #define get_datadir_fname2_suffix(sub1, sub2, suffix) \ options_get_datadir_fname2_suffix(get_options(), (sub1), (sub2), (suffix)) /** Return a newly allocated string containing datadir/sub1. See @@ -115,7 +117,7 @@ int config_parse_commandline(int argc, char **argv, int ignore_errors, config_line_t **cmdline_result); void config_register_addressmaps(const or_options_t *options); -/* XXXX024 move to connection_edge.h */ +/* XXXX move to connection_edge.h */ int addressmap_register_auto(const char *from, const char *to, time_t expires, addressmap_entry_source_t addrmap_source, diff --git a/src/or/confparse.c b/src/or/confparse.c index 4f446d07c3..3532b39d93 100644 --- a/src/or/confparse.c +++ b/src/or/confparse.c @@ -1238,7 +1238,7 @@ config_parse_units(const char *val, struct unit_table_t *u, int *ok) v = tor_parse_uint64(val, 10, 0, UINT64_MAX, ok, &cp); if (!*ok || (cp && *cp == '.')) { - d = tor_parse_double(val, 0, UINT64_MAX, ok, &cp); + d = tor_parse_double(val, 0, (double)UINT64_MAX, ok, &cp); if (!*ok) goto done; use_float = 1; @@ -1255,7 +1255,7 @@ config_parse_units(const char *val, struct unit_table_t *u, int *ok) for ( ;u->unit;++u) { if (!strcasecmp(u->unit, cp)) { if (use_float) - v = u->multiplier * d; + v = (uint64_t)(u->multiplier * d); else v *= u->multiplier; *ok = 1; diff --git a/src/or/connection.c b/src/or/connection.c index 4fbbaf1abd..9eef063f18 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -98,7 +98,7 @@ static int get_proxy_type(void); /** The last addresses that our network interface seemed to have been * binding to. We use this as one way to detect when our IP changes. * - * XXX024 We should really use the entire list of interfaces here. + * XXXX+ We should really use the entire list of interfaces here. **/ static tor_addr_t *last_interface_ipv4 = NULL; /* DOCDOC last_interface_ipv6 */ @@ -665,9 +665,7 @@ connection_free,(connection_t *conn)) return; tor_assert(!connection_is_on_closeable_list(conn)); tor_assert(!connection_in_array(conn)); - if (conn->linked_conn) { - log_err(LD_BUG, "Called with conn->linked_conn still set."); - tor_fragile_assert(); + if (BUG(conn->linked_conn)) { conn->linked_conn->linked_conn = NULL; if (! conn->linked_conn->marked_for_close && conn->linked_conn->reading_from_linked_conn) @@ -1564,7 +1562,7 @@ connection_handle_listener_read(connection_t *conn, int new_type) /* remember the remote address */ tor_addr_copy(&newconn->addr, &addr); newconn->port = port; - newconn->address = tor_dup_addr(&addr); + newconn->address = tor_addr_to_str_dup(&addr); if (new_type == CONN_TYPE_AP && conn->socket_family != AF_UNIX) { log_info(LD_NET, "New SOCKS connection opened from %s.", @@ -2242,7 +2240,7 @@ connection_send_socks5_connect(connection_t *conn) } else { /* AF_INET6 */ buf[3] = 4; reqsize += 16; - memcpy(buf + 4, tor_addr_to_in6(&conn->addr), 16); + memcpy(buf + 4, tor_addr_to_in6_addr8(&conn->addr), 16); memcpy(buf + 20, &port, 2); } @@ -2538,7 +2536,7 @@ retry_listener_ports(smartlist_t *old_conns, real_port, listensockaddr, sizeof(struct sockaddr_storage)); - address = tor_dup_addr(&port->addr); + address = tor_addr_to_str_dup(&port->addr); } if (listensockaddr) { @@ -2699,8 +2697,6 @@ connection_is_rate_limited(connection_t *conn) #ifdef USE_BUFFEREVENTS static struct bufferevent_rate_limit_group *global_rate_limit = NULL; #else -extern int global_read_bucket, global_write_bucket; -extern int global_relayed_read_bucket, global_relayed_write_bucket; /** Did either global write bucket run dry last second? If so, * we are likely to run dry again this second, so be stingy with the @@ -2934,7 +2930,7 @@ static void record_num_bytes_transferred(connection_t *conn, time_t now, size_t num_read, size_t num_written) { - /* XXX024 check if this is necessary */ + /* XXXX check if this is necessary */ if (num_written >= INT_MAX || num_read >= INT_MAX) { log_err(LD_BUG, "Value out of range. num_read=%lu, num_written=%lu, " "connection type=%s, state=%s", @@ -3644,7 +3640,7 @@ connection_read_to_buf(connection_t *conn, ssize_t *max_to_read, * take us over our read allotment, but really we shouldn't be * believing that SSL bytes are the same as TCP bytes anyway. */ int r2 = read_to_buf_tls(or_conn->tls, pending, conn->inbuf); - if (r2<0) { + if (BUG(r2<0)) { log_warn(LD_BUG, "apparently, reading pending bytes can fail."); return -1; } @@ -3761,7 +3757,7 @@ evbuffer_inbuf_callback(struct evbuffer *buf, connection_consider_empty_read_buckets(conn); if (conn->type == CONN_TYPE_AP) { edge_connection_t *edge_conn = TO_EDGE_CONN(conn); - /*XXXX024 check for overflow*/ + /*XXXX++ check for overflow*/ edge_conn->n_read += (int)info->n_added; } } @@ -3782,7 +3778,7 @@ evbuffer_outbuf_callback(struct evbuffer *buf, connection_consider_empty_write_buckets(conn); if (conn->type == CONN_TYPE_AP) { edge_connection_t *edge_conn = TO_EDGE_CONN(conn); - /*XXXX024 check for overflow*/ + /*XXXX++ check for overflow*/ edge_conn->n_written += (int)info->n_deleted; } } @@ -4141,7 +4137,7 @@ connection_handle_write_impl(connection_t *conn, int force) or_conn->bytes_xmitted += result; or_conn->bytes_xmitted_by_tls += n_written; /* So we notice bytes were written even on error */ - /* XXXX024 This cast is safe since we can never write INT_MAX bytes in a + /* XXXX This cast is safe since we can never write INT_MAX bytes in a * single set of TLS operations. But it looks kinda ugly. If we refactor * the *_buf_tls functions, we should make them return ssize_t or size_t * or something. */ diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index 754e9762ea..799baa2acc 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2001 Matej Pfajfar. + /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. * Copyright (c) 2007-2016, The Tor Project, Inc. */ @@ -919,7 +919,7 @@ connection_ap_warn_and_unmark_if_pending_circ(entry_connection_t *entry_conn, /** Tell any AP streams that are waiting for a one-hop tunnel to * <b>failed_digest</b> that they are going to fail. */ -/* XXX024 We should get rid of this function, and instead attach +/* XXXX We should get rid of this function, and instead attach * one-hop streams to circ->p_streams so they get marked in * circuit_mark_for_close like normal p_streams. */ void @@ -1442,7 +1442,7 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); return -1; } - /* XXXX024-1090 Should we also allow foo.bar.exit if ExitNodes is set and + /* XXXX-1090 Should we also allow foo.bar.exit if ExitNodes is set and Bar is not listed in it? I say yes, but our revised manpage branch implies no. */ } @@ -1691,7 +1691,7 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, rend_service_authorization_t *client_auth = rend_client_lookup_service_authorization(socks->address); - const char *cookie = NULL; + const uint8_t *cookie = NULL; rend_auth_type_t auth_type = REND_NO_AUTH; if (client_auth) { log_info(LD_REND, "Using previously configured client authorization " @@ -1703,7 +1703,8 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, /* Fill in the rend_data field so we can start doing a connection to * a hidden service. */ rend_data_t *rend_data = ENTRY_TO_EDGE_CONN(conn)->rend_data = - rend_data_client_create(socks->address, NULL, cookie, auth_type); + rend_data_client_create(socks->address, NULL, (char *) cookie, + auth_type); if (rend_data == NULL) { return -1; } @@ -2290,7 +2291,7 @@ connection_ap_handshake_send_begin(entry_connection_t *ap_conn) edge_conn->stream_id = get_unique_stream_id_by_circ(circ); if (edge_conn->stream_id==0) { - /* XXXX024 Instead of closing this stream, we should make it get + /* XXXX+ Instead of closing this stream, we should make it get * retried on another circuit. */ connection_mark_unattached_ap(ap_conn, END_STREAM_REASON_INTERNAL); @@ -2382,7 +2383,7 @@ connection_ap_handshake_send_resolve(entry_connection_t *ap_conn) edge_conn->stream_id = get_unique_stream_id_by_circ(circ); if (edge_conn->stream_id==0) { - /* XXXX024 Instead of closing this stream, we should make it get + /* XXXX+ Instead of closing this stream, we should make it get * retried on another circuit. */ connection_mark_unattached_ap(ap_conn, END_STREAM_REASON_INTERNAL); @@ -2433,7 +2434,7 @@ connection_ap_handshake_send_resolve(entry_connection_t *ap_conn) if (!base_conn->address) { /* This might be unnecessary. XXXX */ - base_conn->address = tor_dup_addr(&base_conn->addr); + base_conn->address = tor_addr_to_str_dup(&base_conn->addr); } base_conn->state = AP_CONN_STATE_RESOLVE_WAIT; log_info(LD_APP,"Address sent for resolve, ap socket "TOR_SOCKET_T_FORMAT diff --git a/src/or/connection_or.c b/src/or/connection_or.c index 9730e1a952..c69a2ad377 100644 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@ -584,7 +584,7 @@ connection_or_process_inbuf(or_connection_t *conn) * check would otherwise just let data accumulate. It serves no purpose * in 0.2.3. * - * XXX024 Remove this check once we verify that the above paragraph is + * XXXX Remove this check once we verify that the above paragraph is * 100% true. */ if (buf_datalen(conn->base_.inbuf) > MAX_OR_INBUF_WHEN_NONOPEN) { log_fn(LOG_PROTOCOL_WARN, LD_NET, "Accumulated too much data (%d bytes) " @@ -935,7 +935,7 @@ connection_or_init_conn_from_address(or_connection_t *conn, } conn->nickname = tor_strdup(node_get_nickname(r)); tor_free(conn->base_.address); - conn->base_.address = tor_dup_addr(&node_ap.addr); + conn->base_.address = tor_addr_to_str_dup(&node_ap.addr); } else { conn->nickname = tor_malloc(HEX_DIGEST_LEN+2); conn->nickname[0] = '$'; @@ -943,7 +943,7 @@ connection_or_init_conn_from_address(or_connection_t *conn, conn->identity_digest, DIGEST_LEN); tor_free(conn->base_.address); - conn->base_.address = tor_dup_addr(addr); + conn->base_.address = tor_addr_to_str_dup(addr); } /* @@ -1282,11 +1282,9 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port, switch (connection_connect(TO_CONN(conn), conn->base_.address, &addr, port, &socket_error)) { case -1: - /* If the connection failed immediately, and we're using - * a proxy, our proxy is down. Don't blame the Tor server. */ - if (conn->base_.proxy_state == PROXY_INFANT) - entry_guard_register_connect_status(conn->identity_digest, - 0, 1, time(NULL)); + /* We failed to establish a connection probably because of a local + * error. No need to blame the guard in this case. Notify the networking + * system of this failure. */ connection_or_connect_failed(conn, errno_to_orconn_end_reason(socket_error), tor_socket_strerror(socket_error)); diff --git a/src/or/control.c b/src/or/control.c index e2ad8cc6dc..d3613d8d4f 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -190,6 +190,8 @@ static void set_cached_network_liveness(int liveness); static void flush_queued_events_cb(evutil_socket_t fd, short what, void *arg); +static char * download_status_to_string(const download_status_t *dl); + /** Given a control event code for a message event, return the corresponding * log severity. */ static inline int @@ -1211,7 +1213,8 @@ decode_hashed_passwords(config_line_t *passwords) const char *hashed = cl->value; if (!strcmpstart(hashed, "16:")) { - if (base16_decode(decoded, sizeof(decoded), hashed+3, strlen(hashed+3))<0 + if (base16_decode(decoded, sizeof(decoded), hashed+3, strlen(hashed+3)) + != S2K_RFC2440_SPECIFIER_LEN + DIGEST_LEN || strlen(hashed+3) != (S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN)*2) { goto err; } @@ -1262,7 +1265,8 @@ handle_control_authenticate(control_connection_t *conn, uint32_t len, tor_assert(i>0); password_len = i/2; password = tor_malloc(password_len + 1); - if (base16_decode(password, password_len+1, body, i)<0) { + if (base16_decode(password, password_len+1, body, i) + != (int) password_len) { connection_write_str_to_buf( "551 Invalid hexadecimal encoding. Maybe you tried a plain text " "password? If so, the standard requires that you put it in " @@ -1724,8 +1728,6 @@ getinfo_helper_misc(control_connection_t *conn, const char *question, } else if (!strcmp(question, "limits/max-mem-in-queues")) { tor_asprintf(answer, U64_FORMAT, U64_PRINTF_ARG(get_options()->MaxMemInQueues)); - } else if (!strcmp(question, "dir-usage")) { - *answer = directory_dump_request_log(); } else if (!strcmp(question, "fingerprint")) { crypto_pk_t *server_key; if (!server_mode(get_options())) { @@ -1865,7 +1867,7 @@ getinfo_helper_dir(control_connection_t *control_conn, *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len); } } else if (!strcmpstart(question, "desc/name/")) { - /* XXX023 Setting 'warn_if_unnamed' here is a bit silly -- the + /* XXX Setting 'warn_if_unnamed' here is a bit silly -- the * warning goes to the user, not to the controller. */ node = node_get_by_nickname(question+strlen("desc/name/"), 1); if (node) @@ -1951,7 +1953,7 @@ getinfo_helper_dir(control_connection_t *control_conn, *answer = tor_strndup(md->body, md->bodylen); } } else if (!strcmpstart(question, "md/name/")) { - /* XXX023 Setting 'warn_if_unnamed' here is a bit silly -- the + /* XXX Setting 'warn_if_unnamed' here is a bit silly -- the * warning goes to the user, not to the controller. */ const node_t *node = node_get_by_nickname(question+strlen("md/name/"), 1); /* XXXX duplicated code */ @@ -2028,7 +2030,8 @@ getinfo_helper_dir(control_connection_t *control_conn, if (strlen(question) == HEX_DIGEST_LEN) { char d[DIGEST_LEN]; signed_descriptor_t *sd = NULL; - if (base16_decode(d, sizeof(d), question, strlen(question))==0) { + if (base16_decode(d, sizeof(d), question, strlen(question)) + != sizeof(d)) { /* XXXX this test should move into extrainfo_get_by_descriptor_digest, * but I don't want to risk affecting other parts of the code, * especially since the rules for using our own extrainfo (including @@ -2050,6 +2053,411 @@ getinfo_helper_dir(control_connection_t *control_conn, return 0; } +/** Given a smartlist of 20-byte digests, return a newly allocated string + * containing each of those digests in order, formatted in HEX, and terminated + * with a newline. */ +static char * +digest_list_to_string(const smartlist_t *sl) +{ + int len; + char *result, *s; + + /* Allow for newlines, and a \0 at the end */ + len = smartlist_len(sl) * (HEX_DIGEST_LEN + 1) + 1; + result = tor_malloc_zero(len); + + s = result; + SMARTLIST_FOREACH_BEGIN(sl, const char *, digest) { + base16_encode(s, HEX_DIGEST_LEN + 1, digest, DIGEST_LEN); + s[HEX_DIGEST_LEN] = '\n'; + s += HEX_DIGEST_LEN + 1; + } SMARTLIST_FOREACH_END(digest); + *s = '\0'; + + return result; +} + +/** Turn a download_status_t into a human-readable description in a newly + * allocated string. The format is specified in control-spec.txt, under + * the documentation for "GETINFO download/..." . */ +static char * +download_status_to_string(const download_status_t *dl) +{ + char *rv = NULL, *tmp; + char tbuf[ISO_TIME_LEN+1]; + const char *schedule_str, *want_authority_str; + const char *increment_on_str, *backoff_str; + + if (dl) { + /* Get some substrings of the eventual output ready */ + format_iso_time(tbuf, dl->next_attempt_at); + + switch (dl->schedule) { + case DL_SCHED_GENERIC: + schedule_str = "DL_SCHED_GENERIC"; + break; + case DL_SCHED_CONSENSUS: + schedule_str = "DL_SCHED_CONSENSUS"; + break; + case DL_SCHED_BRIDGE: + schedule_str = "DL_SCHED_BRIDGE"; + break; + default: + schedule_str = "unknown"; + break; + } + + switch (dl->want_authority) { + case DL_WANT_ANY_DIRSERVER: + want_authority_str = "DL_WANT_ANY_DIRSERVER"; + break; + case DL_WANT_AUTHORITY: + want_authority_str = "DL_WANT_AUTHORITY"; + break; + default: + want_authority_str = "unknown"; + break; + } + + switch (dl->increment_on) { + case DL_SCHED_INCREMENT_FAILURE: + increment_on_str = "DL_SCHED_INCREMENT_FAILURE"; + break; + case DL_SCHED_INCREMENT_ATTEMPT: + increment_on_str = "DL_SCHED_INCREMENT_ATTEMPT"; + break; + default: + increment_on_str = "unknown"; + break; + } + + switch (dl->backoff) { + case DL_SCHED_DETERMINISTIC: + backoff_str = "DL_SCHED_DETERMINISTIC"; + break; + case DL_SCHED_RANDOM_EXPONENTIAL: + backoff_str = "DL_SCHED_RANDOM_EXPONENTIAL"; + break; + default: + backoff_str = "unknown"; + break; + } + + /* Now assemble them */ + tor_asprintf(&tmp, + "next-attempt-at %s\n" + "n-download-failures %u\n" + "n-download-attempts %u\n" + "schedule %s\n" + "want-authority %s\n" + "increment-on %s\n" + "backoff %s\n", + tbuf, + dl->n_download_failures, + dl->n_download_attempts, + schedule_str, + want_authority_str, + increment_on_str, + backoff_str); + + if (dl->backoff == DL_SCHED_RANDOM_EXPONENTIAL) { + /* Additional fields become relevant in random-exponential mode */ + tor_asprintf(&rv, + "%s" + "last-backoff-position %u\n" + "last-delay-used %d\n", + tmp, + dl->last_backoff_position, + dl->last_delay_used); + tor_free(tmp); + } else { + /* That was it */ + rv = tmp; + } + } + + return rv; +} + +/** Handle the consensus download cases for getinfo_helper_downloads() */ +STATIC void +getinfo_helper_downloads_networkstatus(const char *flavor, + download_status_t **dl_to_emit, + const char **errmsg) +{ + /* + * We get the one for the current bootstrapped status by default, or + * take an extra /bootstrap or /running suffix + */ + if (strcmp(flavor, "ns") == 0) { + *dl_to_emit = networkstatus_get_dl_status_by_flavor(FLAV_NS); + } else if (strcmp(flavor, "ns/bootstrap") == 0) { + *dl_to_emit = networkstatus_get_dl_status_by_flavor_bootstrap(FLAV_NS); + } else if (strcmp(flavor, "ns/running") == 0 ) { + *dl_to_emit = networkstatus_get_dl_status_by_flavor_running(FLAV_NS); + } else if (strcmp(flavor, "microdesc") == 0) { + *dl_to_emit = networkstatus_get_dl_status_by_flavor(FLAV_MICRODESC); + } else if (strcmp(flavor, "microdesc/bootstrap") == 0) { + *dl_to_emit = + networkstatus_get_dl_status_by_flavor_bootstrap(FLAV_MICRODESC); + } else if (strcmp(flavor, "microdesc/running") == 0) { + *dl_to_emit = + networkstatus_get_dl_status_by_flavor_running(FLAV_MICRODESC); + } else { + *errmsg = "Unknown flavor"; + } +} + +/** Handle the cert download cases for getinfo_helper_downloads() */ +STATIC void +getinfo_helper_downloads_cert(const char *fp_sk_req, + download_status_t **dl_to_emit, + smartlist_t **digest_list, + const char **errmsg) +{ + const char *sk_req; + char id_digest[DIGEST_LEN]; + char sk_digest[DIGEST_LEN]; + + /* + * We have to handle four cases; fp_sk_req is the request with + * a prefix of "downloads/cert/" snipped off. + * + * Case 1: fp_sk_req = "fps" + * - We should emit a digest_list with a list of all the identity + * fingerprints that can be queried for certificate download status; + * get it by calling list_authority_ids_with_downloads(). + * + * Case 2: fp_sk_req = "fp/<fp>" for some fingerprint fp + * - We want the default certificate for this identity fingerprint's + * download status; this is the download we get from URLs starting + * in /fp/ on the directory server. We can get it with + * id_only_download_status_for_authority_id(). + * + * Case 3: fp_sk_req = "fp/<fp>/sks" for some fingerprint fp + * - We want a list of all signing key digests for this identity + * fingerprint which can be queried for certificate download status. + * Get it with list_sk_digests_for_authority_id(). + * + * Case 4: fp_sk_req = "fp/<fp>/<sk>" for some fingerprint fp and + * signing key digest sk + * - We want the download status for the certificate for this specific + * signing key and fingerprint. These correspond to the ones we get + * from URLs starting in /fp-sk/ on the directory server. Get it with + * list_sk_digests_for_authority_id(). + */ + + if (strcmp(fp_sk_req, "fps") == 0) { + *digest_list = list_authority_ids_with_downloads(); + if (!(*digest_list)) { + *errmsg = "Failed to get list of authority identity digests (!)"; + } + } else if (!strcmpstart(fp_sk_req, "fp/")) { + fp_sk_req += strlen("fp/"); + /* Okay, look for another / to tell the fp from fp-sk cases */ + sk_req = strchr(fp_sk_req, '/'); + if (sk_req) { + /* okay, split it here and try to parse <fp> */ + if (base16_decode(id_digest, DIGEST_LEN, + fp_sk_req, sk_req - fp_sk_req) == DIGEST_LEN) { + /* Skip past the '/' */ + ++sk_req; + if (strcmp(sk_req, "sks") == 0) { + /* We're asking for the list of signing key fingerprints */ + *digest_list = list_sk_digests_for_authority_id(id_digest); + if (!(*digest_list)) { + *errmsg = "Failed to get list of signing key digests for this " + "authority identity digest"; + } + } else { + /* We've got a signing key digest */ + if (base16_decode(sk_digest, DIGEST_LEN, + sk_req, strlen(sk_req)) == DIGEST_LEN) { + *dl_to_emit = + download_status_for_authority_id_and_sk(id_digest, sk_digest); + if (!(*dl_to_emit)) { + *errmsg = "Failed to get download status for this identity/" + "signing key digest pair"; + } + } else { + *errmsg = "That didn't look like a signing key digest"; + } + } + } else { + *errmsg = "That didn't look like an identity digest"; + } + } else { + /* We're either in downloads/certs/fp/<fp>, or we can't parse <fp> */ + if (strlen(fp_sk_req) == HEX_DIGEST_LEN) { + if (base16_decode(id_digest, DIGEST_LEN, + fp_sk_req, strlen(fp_sk_req)) == DIGEST_LEN) { + *dl_to_emit = id_only_download_status_for_authority_id(id_digest); + if (!(*dl_to_emit)) { + *errmsg = "Failed to get download status for this authority " + "identity digest"; + } + } else { + *errmsg = "That didn't look like a digest"; + } + } else { + *errmsg = "That didn't look like a digest"; + } + } + } else { + *errmsg = "Unknown certificate download status query"; + } +} + +/** Handle the routerdesc download cases for getinfo_helper_downloads() */ +STATIC void +getinfo_helper_downloads_desc(const char *desc_req, + download_status_t **dl_to_emit, + smartlist_t **digest_list, + const char **errmsg) +{ + char desc_digest[DIGEST_LEN]; + /* + * Two cases to handle here: + * + * Case 1: desc_req = "descs" + * - Emit a list of all router descriptor digests, which we get by + * calling router_get_descriptor_digests(); this can return NULL + * if we have no current ns-flavor consensus. + * + * Case 2: desc_req = <fp> + * - Check on the specified fingerprint and emit its download_status_t + * using router_get_dl_status_by_descriptor_digest(). + */ + + if (strcmp(desc_req, "descs") == 0) { + *digest_list = router_get_descriptor_digests(); + if (!(*digest_list)) { + *errmsg = "We don't seem to have a networkstatus-flavored consensus"; + } + /* + * Microdescs don't use the download_status_t mechanism, so we don't + * answer queries about their downloads here; see microdesc.c. + */ + } else if (strlen(desc_req) == HEX_DIGEST_LEN) { + if (base16_decode(desc_digest, DIGEST_LEN, + desc_req, strlen(desc_req)) == DIGEST_LEN) { + /* Okay we got a digest-shaped thing; try asking for it */ + *dl_to_emit = router_get_dl_status_by_descriptor_digest(desc_digest); + if (!(*dl_to_emit)) { + *errmsg = "No such descriptor digest found"; + } + } else { + *errmsg = "That didn't look like a digest"; + } + } else { + *errmsg = "Unknown router descriptor download status query"; + } +} + +/** Handle the bridge download cases for getinfo_helper_downloads() */ +STATIC void +getinfo_helper_downloads_bridge(const char *bridge_req, + download_status_t **dl_to_emit, + smartlist_t **digest_list, + const char **errmsg) +{ + char bridge_digest[DIGEST_LEN]; + /* + * Two cases to handle here: + * + * Case 1: bridge_req = "bridges" + * - Emit a list of all bridge identity digests, which we get by + * calling list_bridge_identities(); this can return NULL if we are + * not using bridges. + * + * Case 2: bridge_req = <fp> + * - Check on the specified fingerprint and emit its download_status_t + * using get_bridge_dl_status_by_id(). + */ + + if (strcmp(bridge_req, "bridges") == 0) { + *digest_list = list_bridge_identities(); + if (!(*digest_list)) { + *errmsg = "We don't seem to be using bridges"; + } + } else if (strlen(bridge_req) == HEX_DIGEST_LEN) { + if (base16_decode(bridge_digest, DIGEST_LEN, + bridge_req, strlen(bridge_req)) == DIGEST_LEN) { + /* Okay we got a digest-shaped thing; try asking for it */ + *dl_to_emit = get_bridge_dl_status_by_id(bridge_digest); + if (!(*dl_to_emit)) { + *errmsg = "No such bridge identity digest found"; + } + } else { + *errmsg = "That didn't look like a digest"; + } + } else { + *errmsg = "Unknown bridge descriptor download status query"; + } +} + +/** Implementation helper for GETINFO: knows the answers for questions about + * download status information. */ +STATIC int +getinfo_helper_downloads(control_connection_t *control_conn, + const char *question, char **answer, + const char **errmsg) +{ + download_status_t *dl_to_emit = NULL; + smartlist_t *digest_list = NULL; + + /* Assert args are sane */ + tor_assert(control_conn != NULL); + tor_assert(question != NULL); + tor_assert(answer != NULL); + tor_assert(errmsg != NULL); + + /* We check for this later to see if we should supply a default */ + *errmsg = NULL; + + /* Are we after networkstatus downloads? */ + if (!strcmpstart(question, "downloads/networkstatus/")) { + getinfo_helper_downloads_networkstatus( + question + strlen("downloads/networkstatus/"), + &dl_to_emit, errmsg); + /* Certificates? */ + } else if (!strcmpstart(question, "downloads/cert/")) { + getinfo_helper_downloads_cert( + question + strlen("downloads/cert/"), + &dl_to_emit, &digest_list, errmsg); + /* Router descriptors? */ + } else if (!strcmpstart(question, "downloads/desc/")) { + getinfo_helper_downloads_desc( + question + strlen("downloads/desc/"), + &dl_to_emit, &digest_list, errmsg); + /* Bridge descriptors? */ + } else if (!strcmpstart(question, "downloads/bridge/")) { + getinfo_helper_downloads_bridge( + question + strlen("downloads/bridge/"), + &dl_to_emit, &digest_list, errmsg); + } else { + *errmsg = "Unknown download status query"; + } + + if (dl_to_emit) { + *answer = download_status_to_string(dl_to_emit); + + return 0; + } else if (digest_list) { + *answer = digest_list_to_string(digest_list); + SMARTLIST_FOREACH(digest_list, void *, s, tor_free(s)); + smartlist_free(digest_list); + + return 0; + } else { + if (!(*errmsg)) { + *errmsg = "Unknown error"; + } + + return -1; + } +} + /** Allocate and return a description of <b>circ</b>'s current status, * including its path (if any). */ static char * @@ -2489,6 +2897,49 @@ static const getinfo_item_t getinfo_items[] = { DOC("config/defaults", "List of default values for configuration options. " "See also config/names"), + PREFIX("downloads/networkstatus/", downloads, + "Download statuses for networkstatus objects"), + DOC("downloads/networkstatus/ns", + "Download status for current-mode networkstatus download"), + DOC("downloads/networkstatus/ns/bootstrap", + "Download status for bootstrap-time networkstatus download"), + DOC("downloads/networkstatus/ns/running", + "Download status for run-time networkstatus download"), + DOC("downloads/networkstatus/microdesc", + "Download status for current-mode microdesc download"), + DOC("downloads/networkstatus/microdesc/bootstrap", + "Download status for bootstrap-time microdesc download"), + DOC("downloads/networkstatus/microdesc/running", + "Download status for run-time microdesc download"), + PREFIX("downloads/cert/", downloads, + "Download statuses for certificates, by id fingerprint and " + "signing key"), + DOC("downloads/cert/fps", + "List of authority fingerprints for which any download statuses " + "exist"), + DOC("downloads/cert/fp/<fp>", + "Download status for <fp> with the default signing key; corresponds " + "to /fp/ URLs on directory server."), + DOC("downloads/cert/fp/<fp>/sks", + "List of signing keys for which specific download statuses are " + "available for this id fingerprint"), + DOC("downloads/cert/fp/<fp>/<sk>", + "Download status for <fp> with signing key <sk>; corresponds " + "to /fp-sk/ URLs on directory server."), + PREFIX("downloads/desc/", downloads, + "Download statuses for router descriptors, by descriptor digest"), + DOC("downloads/desc/descs", + "Return a list of known router descriptor digests"), + DOC("downloads/desc/<desc>", + "Return a download status for a given descriptor digest"), + PREFIX("downloads/bridge/", downloads, + "Download statuses for bridge descriptors, by bridge identity " + "digest"), + DOC("downloads/bridge/bridges", + "Return a list of configured bridge identity digests with download " + "statuses"), + DOC("downloads/bridge/<desc>", + "Return a download status for a given bridge identity digest"), ITEM("info/names", misc, "List of GETINFO options, types, and documentation."), ITEM("events/names", misc, @@ -2561,7 +3012,6 @@ static const getinfo_item_t getinfo_items[] = { "Username under which the tor process is running."), ITEM("process/descriptor-limit", misc, "File descriptor limit."), ITEM("limits/max-mem-in-queues", misc, "Actual limit on memory in queues"), - ITEM("dir-usage", misc, "Breakdown of bytes transferred over DirPort."), PREFIX("desc-annotations/id/", dir, "Router annotations by hexdigest."), PREFIX("dir/server/", dir,"Router descriptors as retrieved from a DirPort."), PREFIX("dir/status/", dir, @@ -3445,7 +3895,8 @@ handle_control_authchallenge(control_connection_t *conn, uint32_t len, client_nonce = tor_malloc_zero(client_nonce_len); if (base16_decode(client_nonce, client_nonce_len, - cp, client_nonce_encoded_len) < 0) { + cp, client_nonce_encoded_len) + != (int) client_nonce_len) { connection_write_str_to_buf("513 Invalid base16 client nonce\r\n", conn); connection_mark_for_close(TO_CONN(conn)); @@ -3791,14 +4242,18 @@ handle_control_add_onion(control_connection_t *conn, * the other arguments are malformed. */ smartlist_t *port_cfgs = smartlist_new(); + smartlist_t *auth_clients = NULL; + smartlist_t *auth_created_clients = NULL; int discard_pk = 0; int detach = 0; int max_streams = 0; int max_streams_close_circuit = 0; + rend_auth_type_t auth_type = REND_NO_AUTH; for (size_t i = 1; i < arg_len; i++) { static const char *port_prefix = "Port="; static const char *flags_prefix = "Flags="; static const char *max_s_prefix = "MaxStreams="; + static const char *auth_prefix = "ClientAuth="; const char *arg = smartlist_get(args, i); if (!strcasecmpstart(arg, port_prefix)) { @@ -3829,10 +4284,12 @@ handle_control_add_onion(control_connection_t *conn, * connection. * * 'MaxStreamsCloseCircuit' - Close the circuit if MaxStreams is * exceeded. + * * 'BasicAuth' - Client authorization using the 'basic' method. */ static const char *discard_flag = "DiscardPK"; static const char *detach_flag = "Detach"; static const char *max_s_close_flag = "MaxStreamsCloseCircuit"; + static const char *basicauth_flag = "BasicAuth"; smartlist_t *flags = smartlist_new(); int bad = 0; @@ -3851,6 +4308,8 @@ handle_control_add_onion(control_connection_t *conn, detach = 1; } else if (!strcasecmp(flag, max_s_close_flag)) { max_streams_close_circuit = 1; + } else if (!strcasecmp(flag, basicauth_flag)) { + auth_type = REND_BASIC_AUTH; } else { connection_printf_to_buf(conn, "512 Invalid 'Flags' argument: %s\r\n", @@ -3863,6 +4322,42 @@ handle_control_add_onion(control_connection_t *conn, smartlist_free(flags); if (bad) goto out; + } else if (!strcasecmpstart(arg, auth_prefix)) { + char *err_msg = NULL; + int created = 0; + rend_authorized_client_t *client = + add_onion_helper_clientauth(arg + strlen(auth_prefix), + &created, &err_msg); + if (!client) { + if (err_msg) { + connection_write_str_to_buf(err_msg, conn); + tor_free(err_msg); + } + goto out; + } + + if (auth_clients != NULL) { + int bad = 0; + SMARTLIST_FOREACH_BEGIN(auth_clients, rend_authorized_client_t *, ac) { + if (strcmp(ac->client_name, client->client_name) == 0) { + bad = 1; + break; + } + } SMARTLIST_FOREACH_END(ac); + if (bad) { + connection_printf_to_buf(conn, + "512 Duplicate name in ClientAuth\r\n"); + rend_authorized_client_free(client); + goto out; + } + } else { + auth_clients = smartlist_new(); + auth_created_clients = smartlist_new(); + } + smartlist_add(auth_clients, client); + if (created) { + smartlist_add(auth_created_clients, client); + } } else { connection_printf_to_buf(conn, "513 Invalid argument\r\n"); goto out; @@ -3871,6 +4366,18 @@ handle_control_add_onion(control_connection_t *conn, if (smartlist_len(port_cfgs) == 0) { connection_printf_to_buf(conn, "512 Missing 'Port' argument\r\n"); goto out; + } else if (auth_type == REND_NO_AUTH && auth_clients != NULL) { + connection_printf_to_buf(conn, "512 No auth type specified\r\n"); + goto out; + } else if (auth_type != REND_NO_AUTH && auth_clients == NULL) { + connection_printf_to_buf(conn, "512 No auth clients specified\r\n"); + goto out; + } else if ((auth_type == REND_BASIC_AUTH && + smartlist_len(auth_clients) > 512) || + (auth_type == REND_STEALTH_AUTH && + smartlist_len(auth_clients) > 16)) { + connection_printf_to_buf(conn, "512 Too many auth clients\r\n"); + goto out; } /* Parse the "keytype:keyblob" argument. */ @@ -3891,35 +4398,21 @@ handle_control_add_onion(control_connection_t *conn, } tor_assert(!err_msg); - /* Create the HS, using private key pk, and port config port_cfg. + /* Create the HS, using private key pk, client authentication auth_type, + * the list of auth_clients, and port config port_cfg. * rend_service_add_ephemeral() will take ownership of pk and port_cfg, * regardless of success/failure. */ char *service_id = NULL; int ret = rend_service_add_ephemeral(pk, port_cfgs, max_streams, max_streams_close_circuit, + auth_type, auth_clients, &service_id); port_cfgs = NULL; /* port_cfgs is now owned by the rendservice code. */ + auth_clients = NULL; /* so is auth_clients */ switch (ret) { case RSAE_OKAY: { - char *buf = NULL; - tor_assert(service_id); - if (key_new_alg) { - tor_assert(key_new_blob); - tor_asprintf(&buf, - "250-ServiceID=%s\r\n" - "250-PrivateKey=%s:%s\r\n" - "250 OK\r\n", - service_id, - key_new_alg, - key_new_blob); - } else { - tor_asprintf(&buf, - "250-ServiceID=%s\r\n" - "250 OK\r\n", - service_id); - } if (detach) { if (!detached_onion_services) detached_onion_services = smartlist_new(); @@ -3930,9 +4423,26 @@ handle_control_add_onion(control_connection_t *conn, smartlist_add(conn->ephemeral_onion_services, service_id); } - connection_write_str_to_buf(buf, conn); - memwipe(buf, 0, strlen(buf)); - tor_free(buf); + tor_assert(service_id); + connection_printf_to_buf(conn, "250-ServiceID=%s\r\n", service_id); + if (key_new_alg) { + tor_assert(key_new_blob); + connection_printf_to_buf(conn, "250-PrivateKey=%s:%s\r\n", + key_new_alg, key_new_blob); + } + if (auth_created_clients) { + SMARTLIST_FOREACH(auth_created_clients, rend_authorized_client_t *, ac, { + char *encoded = rend_auth_encode_cookie(ac->descriptor_cookie, + auth_type); + tor_assert(encoded); + connection_printf_to_buf(conn, "250-ClientAuth=%s:%s\r\n", + ac->client_name, encoded); + memwipe(encoded, 0, strlen(encoded)); + tor_free(encoded); + }); + } + + connection_printf_to_buf(conn, "250 OK\r\n"); break; } case RSAE_BADPRIVKEY: @@ -3944,6 +4454,9 @@ handle_control_add_onion(control_connection_t *conn, case RSAE_BADVIRTPORT: connection_printf_to_buf(conn, "512 Invalid VIRTPORT/TARGET\r\n"); break; + case RSAE_BADAUTH: + connection_printf_to_buf(conn, "512 Invalid client authorization\r\n"); + break; case RSAE_INTERNAL: /* FALLSTHROUGH */ default: connection_printf_to_buf(conn, "551 Failed to add Onion Service\r\n"); @@ -3960,6 +4473,16 @@ handle_control_add_onion(control_connection_t *conn, smartlist_free(port_cfgs); } + if (auth_clients) { + SMARTLIST_FOREACH(auth_clients, rend_authorized_client_t *, ac, + rend_authorized_client_free(ac)); + smartlist_free(auth_clients); + } + if (auth_created_clients) { + // Do not free entries; they are the same as auth_clients + smartlist_free(auth_created_clients); + } + SMARTLIST_FOREACH(args, char *, cp, { memwipe(cp, 0, strlen(cp)); tor_free(cp); @@ -4068,6 +4591,65 @@ add_onion_helper_keyarg(const char *arg, int discard_pk, return pk; } +/** Helper function to handle parsing a ClientAuth argument to the + * ADD_ONION command. Return a new rend_authorized_client_t, or NULL + * and an optional control protocol error message on failure. The + * caller is responsible for freeing the returned auth_client and err_msg. + * + * If 'created' is specified, it will be set to 1 when a new cookie has + * been generated. + */ +STATIC rend_authorized_client_t * +add_onion_helper_clientauth(const char *arg, int *created, char **err_msg) +{ + int ok = 0; + + tor_assert(arg); + tor_assert(created); + tor_assert(err_msg); + *err_msg = NULL; + + smartlist_t *auth_args = smartlist_new(); + rend_authorized_client_t *client = + tor_malloc_zero(sizeof(rend_authorized_client_t)); + smartlist_split_string(auth_args, arg, ":", 0, 0); + if (smartlist_len(auth_args) < 1 || smartlist_len(auth_args) > 2) { + *err_msg = tor_strdup("512 Invalid ClientAuth syntax\r\n"); + goto err; + } + client->client_name = tor_strdup(smartlist_get(auth_args, 0)); + if (smartlist_len(auth_args) == 2) { + char *decode_err_msg = NULL; + if (rend_auth_decode_cookie(smartlist_get(auth_args, 1), + client->descriptor_cookie, + NULL, &decode_err_msg) < 0) { + tor_assert(decode_err_msg); + tor_asprintf(err_msg, "512 %s\r\n", decode_err_msg); + tor_free(decode_err_msg); + goto err; + } + *created = 0; + } else { + crypto_rand((char *) client->descriptor_cookie, REND_DESC_COOKIE_LEN); + *created = 1; + } + + if (!rend_valid_client_name(client->client_name)) { + *err_msg = tor_strdup("512 Invalid name in ClientAuth\r\n"); + goto err; + } + + ok = 1; + err: + SMARTLIST_FOREACH(auth_args, char *, arg, tor_free(arg)); + smartlist_free(auth_args); + if (!ok) { + rend_authorized_client_free(client); + client = NULL; + } + return client; +} + /** Called when we get a DEL_ONION command; parse the body, and remove * the existing ephemeral Onion Service. */ static int diff --git a/src/or/control.h b/src/or/control.h index 008bfb1c3b..6330c85571 100644 --- a/src/or/control.h +++ b/src/or/control.h @@ -259,6 +259,33 @@ STATIC crypto_pk_t *add_onion_helper_keyarg(const char *arg, int discard_pk, const char **key_new_alg_out, char **key_new_blob_out, char **err_msg_out); +STATIC rend_authorized_client_t * +add_onion_helper_clientauth(const char *arg, int *created, char **err_msg_out); + +STATIC void getinfo_helper_downloads_networkstatus( + const char *flavor, + download_status_t **dl_to_emit, + const char **errmsg); +STATIC void getinfo_helper_downloads_cert( + const char *fp_sk_req, + download_status_t **dl_to_emit, + smartlist_t **digest_list, + const char **errmsg); +STATIC void getinfo_helper_downloads_desc( + const char *desc_req, + download_status_t **dl_to_emit, + smartlist_t **digest_list, + const char **errmsg); +STATIC void getinfo_helper_downloads_bridge( + const char *bridge_req, + download_status_t **dl_to_emit, + smartlist_t **digest_list, + const char **errmsg); +STATIC int getinfo_helper_downloads( + control_connection_t *control_conn, + const char *question, char **answer, + const char **errmsg); + #endif #endif diff --git a/src/or/dircollate.c b/src/or/dircollate.c index 3f9d78f02d..756011b934 100644 --- a/src/or/dircollate.c +++ b/src/or/dircollate.c @@ -67,9 +67,9 @@ ddmap_entry_set_digests(ddmap_entry_t *ent, } HT_PROTOTYPE(double_digest_map, ddmap_entry_s, node, ddmap_entry_hash, - ddmap_entry_eq); + ddmap_entry_eq) HT_GENERATE2(double_digest_map, ddmap_entry_s, node, ddmap_entry_hash, - ddmap_entry_eq, 0.6, tor_reallocarray, tor_free_); + ddmap_entry_eq, 0.6, tor_reallocarray, tor_free_) /** Helper: add a single vote_routerstatus_t <b>vrs</b> to the collator * <b>dc</b>, indexing it by its RSA key digest, and by the 2-tuple of diff --git a/src/or/directory.c b/src/or/directory.c index 89b08223d2..7c1b9884c0 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -30,6 +30,7 @@ #include "routerlist.h" #include "routerparse.h" #include "routerset.h" +#include "shared_random.h" #if defined(EXPORTMALLINFO) && defined(HAVE_MALLOC_H) && defined(HAVE_MALLINFO) #ifndef OPENBSD @@ -80,7 +81,6 @@ static void dir_routerdesc_download_failed(smartlist_t *failed, int was_descriptor_digests); static void dir_microdesc_download_failed(smartlist_t *failed, int status_code); -static void note_client_request(int purpose, int compressed, size_t bytes); static int client_likes_consensus(networkstatus_t *v, const char *want_url); static void directory_initiate_command_rend( @@ -123,7 +123,7 @@ static void connection_dir_close_consensus_fetches( /** Return true iff the directory purpose <b>dir_purpose</b> (and if it's * fetching descriptors, it's fetching them for <b>router_purpose</b>) * must use an anonymous connection to a directory. */ -STATIC int +int purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose) { if (get_options()->AllDirActionsPrivate) @@ -495,8 +495,9 @@ MOCK_IMPL(void, directory_get_from_dirserver, ( * sort of dir fetch we'll be doing, so it won't return a bridge * that can't answer our question. */ - /* XXX024 Not all bridges handle conditional consensus downloading, - * so, for now, never assume the server supports that. -PP */ + /* XXX+++++ Not all bridges handle conditional consensus downloading, + * so, for now, never assume the server supports that. -PP + * Is that assumption still so in 2016? -NM */ const node_t *node = choose_random_dirguard(type); if (node && node->ri) { /* every bridge has a routerinfo. */ @@ -727,6 +728,10 @@ directory_initiate_command_routerstatus_rend(const routerstatus_t *status, node = node_get_by_id(status->identity_digest); + /* XXX The below check is wrong: !node means it's not in the consensus, + * but we haven't checked if we have a descriptor for it -- and also, + * we only care about the descriptor if it's a begindir-style anonymized + * connection. */ if (!node && anonymized_connection) { log_info(LD_DIR, "Not sending anonymized request to directory '%s'; we " "don't have its router descriptor.", @@ -744,7 +749,7 @@ directory_initiate_command_routerstatus_rend(const routerstatus_t *status, return; } - /* At this point, if we are a clients making a direct connection to a + /* At this point, if we are a client making a direct connection to a * directory server, we have selected a server that has at least one address * allowed by ClientUseIPv4/6 and Reachable{"",OR,Dir}Addresses. This * selection uses the preference in ClientPreferIPv6{OR,Dir}Port, if @@ -869,7 +874,7 @@ connection_dir_retry_bridges(smartlist_t *descs) char digest[DIGEST_LEN]; SMARTLIST_FOREACH(descs, const char *, cp, { - if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp))<0) { + if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp)) != DIGEST_LEN) { log_warn(LD_BUG, "Malformed fingerprint in list: %s", escaped(cp)); continue; @@ -1178,7 +1183,7 @@ directory_initiate_command_rend(const tor_addr_port_t *or_addr_port, /* set up conn so it's got all the data we need to remember */ tor_addr_copy(&conn->base_.addr, &addr); conn->base_.port = port; - conn->base_.address = tor_dup_addr(&addr); + conn->base_.address = tor_addr_to_str_dup(&addr); memcpy(conn->identity_digest, digest, DIGEST_LEN); conn->base_.purpose = dir_purpose; @@ -1839,7 +1844,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) char *body; char *headers; char *reason = NULL; - size_t body_len = 0, orig_len = 0; + size_t body_len = 0; int status_code; time_t date_header = 0; long apparent_skew; @@ -1849,7 +1854,6 @@ connection_dir_client_reached_eof(dir_connection_t *conn) int allow_partial = (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC || conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO || conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC); - int was_compressed = 0; time_t now = time(NULL); int src_code; @@ -1868,7 +1872,6 @@ connection_dir_client_reached_eof(dir_connection_t *conn) return -1; /* case 1, fall through */ } - orig_len = body_len; if (parse_http_response(headers, &status_code, &date_header, &compression, &reason) < 0) { @@ -1986,7 +1989,6 @@ connection_dir_client_reached_eof(dir_connection_t *conn) tor_free(body); body = new_body; body_len = new_len; - was_compressed = 1; } } @@ -2006,7 +2008,8 @@ connection_dir_client_reached_eof(dir_connection_t *conn) } log_info(LD_DIR,"Received consensus directory (size %d) from server " "'%s:%d'", (int)body_len, conn->base_.address, conn->base_.port); - if ((r=networkstatus_set_current_consensus(body, flavname, 0))<0) { + if ((r=networkstatus_set_current_consensus(body, flavname, 0, + conn->identity_digest))<0) { log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR, "Unable to load %s consensus directory downloaded from " "server '%s:%d'. I'll try again soon.", @@ -2024,6 +2027,10 @@ connection_dir_client_reached_eof(dir_connection_t *conn) update_microdescs_from_networkstatus(now); update_microdesc_downloads(now); directory_info_has_arrived(now, 0, 0); + if (authdir_mode_v3(get_options())) { + sr_act_post_consensus( + networkstatus_get_latest_consensus_by_flavor(FLAV_NS)); + } log_info(LD_DIR, "Successfully loaded consensus."); } @@ -2053,7 +2060,8 @@ connection_dir_client_reached_eof(dir_connection_t *conn) } if (src_code != -1) { - if (trusted_dirs_load_certs_from_string(body, src_code, 1)<0) { + if (trusted_dirs_load_certs_from_string(body, src_code, 1, + conn->identity_digest)<0) { log_warn(LD_DIR, "Unable to parse fetched certificates"); /* if we fetched more than one and only some failed, the successful * ones got flushed to disk so it's safe to call this on them */ @@ -2249,7 +2257,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) ds->nickname); /* XXXX use this information; be sure to upload next one * sooner. -NM */ - /* XXXX023 On further thought, the task above implies that we're + /* XXXX++ On further thought, the task above implies that we're * basing our regenerate-descriptor time on when we uploaded the * last descriptor, not on the published time of the last * descriptor. If those are different, that's a bad thing to @@ -2450,7 +2458,6 @@ connection_dir_client_reached_eof(dir_connection_t *conn) break; } } - note_client_request(conn->base_.purpose, was_compressed, orig_len); tor_free(body); tor_free(headers); tor_free(reason); return 0; } @@ -2651,129 +2658,6 @@ write_http_response_header(dir_connection_t *conn, ssize_t length, cache_lifetime); } -#if defined(INSTRUMENT_DOWNLOADS) || defined(RUNNING_DOXYGEN) -/* DOCDOC */ -typedef struct request_t { - uint64_t bytes; /**< How many bytes have we transferred? */ - uint64_t count; /**< How many requests have we made? */ -} request_t; - -/** Map used to keep track of how much data we've up/downloaded in what kind - * of request. Maps from request type to pointer to request_t. */ -static strmap_t *request_map = NULL; - -/** Record that a client request of <b>purpose</b> was made, and that - * <b>bytes</b> bytes of possibly <b>compressed</b> data were sent/received. - * Used to keep track of how much we've up/downloaded in what kind of - * request. */ -static void -note_client_request(int purpose, int compressed, size_t bytes) -{ - char *key; - const char *kind = NULL; - switch (purpose) { - case DIR_PURPOSE_FETCH_CONSENSUS: kind = "dl/consensus"; break; - case DIR_PURPOSE_FETCH_CERTIFICATE: kind = "dl/cert"; break; - case DIR_PURPOSE_FETCH_STATUS_VOTE: kind = "dl/vote"; break; - case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES: kind = "dl/detached_sig"; - break; - case DIR_PURPOSE_FETCH_SERVERDESC: kind = "dl/server"; break; - case DIR_PURPOSE_FETCH_EXTRAINFO: kind = "dl/extra"; break; - case DIR_PURPOSE_UPLOAD_DIR: kind = "dl/ul-dir"; break; - case DIR_PURPOSE_UPLOAD_VOTE: kind = "dl/ul-vote"; break; - case DIR_PURPOSE_UPLOAD_SIGNATURES: kind = "dl/ul-sig"; break; - case DIR_PURPOSE_FETCH_RENDDESC_V2: kind = "dl/rend2"; break; - case DIR_PURPOSE_UPLOAD_RENDDESC_V2: kind = "dl/ul-rend2"; break; - } - if (kind) { - tor_asprintf(&key, "%s%s", kind, compressed?".z":""); - } else { - tor_asprintf(&key, "unknown purpose (%d)%s", - purpose, compressed?".z":""); - } - note_request(key, bytes); - tor_free(key); -} - -/** Helper: initialize the request map to instrument downloads. */ -static void -ensure_request_map_initialized(void) -{ - if (!request_map) - request_map = strmap_new(); -} - -/** Called when we just transmitted or received <b>bytes</b> worth of data - * because of a request of type <b>key</b> (an arbitrary identifier): adds - * <b>bytes</b> to the total associated with key. */ -void -note_request(const char *key, size_t bytes) -{ - request_t *r; - ensure_request_map_initialized(); - - r = strmap_get(request_map, key); - if (!r) { - r = tor_malloc_zero(sizeof(request_t)); - strmap_set(request_map, key, r); - } - r->bytes += bytes; - r->count++; -} - -/** Return a newly allocated string holding a summary of bytes used per - * request type. */ -char * -directory_dump_request_log(void) -{ - smartlist_t *lines; - char *result; - strmap_iter_t *iter; - - ensure_request_map_initialized(); - - lines = smartlist_new(); - - for (iter = strmap_iter_init(request_map); - !strmap_iter_done(iter); - iter = strmap_iter_next(request_map, iter)) { - const char *key; - void *val; - request_t *r; - strmap_iter_get(iter, &key, &val); - r = val; - smartlist_add_asprintf(lines, "%s "U64_FORMAT" "U64_FORMAT"\n", - key, U64_PRINTF_ARG(r->bytes), U64_PRINTF_ARG(r->count)); - } - smartlist_sort_strings(lines); - result = smartlist_join_strings(lines, "", 0, NULL); - SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp)); - smartlist_free(lines); - return result; -} -#else -static void -note_client_request(int purpose, int compressed, size_t bytes) -{ - (void)purpose; - (void)compressed; - (void)bytes; -} - -void -note_request(const char *key, size_t bytes) -{ - (void)key; - (void)bytes; -} - -char * -directory_dump_request_log(void) -{ - return tor_strdup("Not supported."); -} -#endif - /** Decide whether a client would accept the consensus we have. * * Clients can say they only want a consensus if it's signed by more @@ -2803,7 +2687,8 @@ client_likes_consensus(networkstatus_t *v, const char *want_url) if (want_len > DIGEST_LEN) want_len = DIGEST_LEN; - if (base16_decode(want_digest, DIGEST_LEN, d, want_len*2) < 0) { + if (base16_decode(want_digest, DIGEST_LEN, d, want_len*2) + != (int) want_len) { log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Failed to decode requested authority digest %s.", escaped(d)); continue; @@ -2845,18 +2730,81 @@ choose_compression_level(ssize_t n_bytes) } } +/** Information passed to handle a GET request. */ +typedef struct get_handler_args_t { + /** True if the client asked for compressed data. */ + int compressed; + /** If nonzero, the time included an if-modified-since header with this + * value. */ + time_t if_modified_since; + /** String containing the requested URL or resource. */ + const char *url; + /** String containing the HTTP headers */ + const char *headers; +} get_handler_args_t; + +/** Entry for handling an HTTP GET request. + * + * This entry matches a request if "string" is equal to the requested + * resource, or if "is_prefix" is true and "string" is a prefix of the + * requested resource. + * + * The 'handler' function is called to handle the request. It receives + * an arguments structure, and must return 0 on success or -1 if we should + * close the connection. + **/ +typedef struct url_table_ent_s { + const char *string; + int is_prefix; + int (*handler)(dir_connection_t *conn, const get_handler_args_t *args); +} url_table_ent_t; + +static int handle_get_frontpage(dir_connection_t *conn, + const get_handler_args_t *args); +static int handle_get_current_consensus(dir_connection_t *conn, + const get_handler_args_t *args); +static int handle_get_status_vote(dir_connection_t *conn, + const get_handler_args_t *args); +static int handle_get_microdesc(dir_connection_t *conn, + const get_handler_args_t *args); +static int handle_get_descriptor(dir_connection_t *conn, + const get_handler_args_t *args); +static int handle_get_keys(dir_connection_t *conn, + const get_handler_args_t *args); +static int handle_get_rendezvous2(dir_connection_t *conn, + const get_handler_args_t *args); +static int handle_get_robots(dir_connection_t *conn, + const get_handler_args_t *args); +static int handle_get_networkstatus_bridges(dir_connection_t *conn, + const get_handler_args_t *args); + +/** Table for handling GET requests. */ +static const url_table_ent_t url_table[] = { + { "/tor/", 0, handle_get_frontpage }, + { "/tor/status-vote/current/consensus", 1, handle_get_current_consensus }, + { "/tor/status-vote/current/", 1, handle_get_status_vote }, + { "/tor/status-vote/next/", 1, handle_get_status_vote }, + { "/tor/micro/d/", 1, handle_get_microdesc }, + { "/tor/server/", 1, handle_get_descriptor }, + { "/tor/extra/", 1, handle_get_descriptor }, + { "/tor/keys/", 1, handle_get_keys }, + { "/tor/rendezvous2/", 1, handle_get_rendezvous2 }, + { "/tor/robots.txt", 0, handle_get_robots }, + { "/tor/networkstatus-bridges", 0, handle_get_networkstatus_bridges }, + { NULL, 0, NULL }, +}; + /** Helper function: called when a dirserver gets a complete HTTP GET * request. Look for a request for a directory or for a rendezvous * service descriptor. On finding one, write a response into - * conn-\>outbuf. If the request is unrecognized, send a 400. - * Always return 0. */ + * conn-\>outbuf. If the request is unrecognized, send a 404. + * Return 0 if we handled this successfully, or -1 if we need to close + * the connection. */ STATIC int directory_handle_command_get(dir_connection_t *conn, const char *headers, const char *req_body, size_t req_body_len) { - size_t dlen; char *url, *url_mem, *header; - const or_options_t *options = get_options(); time_t if_modified_since = 0; int compressed; size_t url_len; @@ -2896,29 +2844,73 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, url_len -= 2; } - if (!strcmp(url,"/tor/")) { - const char *frontpage = get_dirportfrontpage(); - - if (frontpage) { - dlen = strlen(frontpage); - /* Let's return a disclaimer page (users shouldn't use V1 anymore, - and caches don't fetch '/', so this is safe). */ - - /* [We don't check for write_bucket_low here, since we want to serve - * this page no matter what.] */ - note_request(url, dlen); - write_http_response_header_impl(conn, dlen, "text/html", "identity", - NULL, DIRPORTFRONTPAGE_CACHE_LIFETIME); - connection_write_to_buf(frontpage, dlen, TO_CONN(conn)); + get_handler_args_t args; + args.url = url; + args.headers = headers; + args.if_modified_since = if_modified_since; + args.compressed = compressed; + + int i, result = -1; + for (i = 0; url_table[i].string; ++i) { + int match; + if (url_table[i].is_prefix) { + match = !strcmpstart(url, url_table[i].string); + } else { + match = !strcmp(url, url_table[i].string); + } + if (match) { + result = url_table[i].handler(conn, &args); goto done; } - /* if no disclaimer file, fall through and continue */ } - if (!strcmpstart(url, "/tor/status-vote/current/consensus")) { + /* we didn't recognize the url */ + write_http_status_line(conn, 404, "Not found"); + result = 0; + + done: + tor_free(url_mem); + return result; +} + +/** Helper function for GET / or GET /tor/ + */ +static int +handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args) +{ + (void) args; /* unused */ + const char *frontpage = get_dirportfrontpage(); + + if (frontpage) { + size_t dlen; + dlen = strlen(frontpage); + /* Let's return a disclaimer page (users shouldn't use V1 anymore, + and caches don't fetch '/', so this is safe). */ + + /* [We don't check for write_bucket_low here, since we want to serve + * this page no matter what.] */ + write_http_response_header_impl(conn, dlen, "text/html", "identity", + NULL, DIRPORTFRONTPAGE_CACHE_LIFETIME); + connection_write_to_buf(frontpage, dlen, TO_CONN(conn)); + } else { + write_http_status_line(conn, 404, "Not found"); + } + return 0; +} + +/** Helper function for GET /tor/status-vote/current/consensus + */ +static int +handle_get_current_consensus(dir_connection_t *conn, + const get_handler_args_t *args) +{ + const char *url = args->url; + const int compressed = args->compressed; + const time_t if_modified_since = args->if_modified_since; + + { /* v3 network status fetch. */ smartlist_t *dir_fps = smartlist_new(); - const char *request_type = NULL; long lifetime = NETWORKSTATUS_CACHE_LIFETIME; if (1) { @@ -2967,7 +2959,6 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, tor_free(flavor); smartlist_add(dir_fps, fp); } - request_type = compressed?"v3.z":"v3"; lifetime = (v && v->fresh_until > now) ? v->fresh_until - now : 0; } @@ -2992,7 +2983,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, goto done; } - dlen = dirserv_estimate_data_size(dir_fps, 0, compressed); + size_t dlen = dirserv_estimate_data_size(dir_fps, 0, compressed); if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) { log_debug(LD_DIRSERV, "Client asked for network status lists, but we've been " @@ -3022,8 +3013,6 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } } - // note_request(request_type,dlen); - (void) request_type; write_http_response_header(conn, -1, compressed, smartlist_len(dir_fps) == 1 ? lifetime : 0); conn->fingerprint_stack = dir_fps; @@ -3036,17 +3025,24 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, goto done; } - if (!strcmpstart(url,"/tor/status-vote/current/") || - !strcmpstart(url,"/tor/status-vote/next/")) { - /* XXXX If-modified-since is only implemented for the current - * consensus: that's probably fine, since it's the only vote document - * people fetch much. */ + done: + return 0; +} + +/** Helper function for GET /tor/status-vote/{current,next}/... + */ +static int +handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) +{ + const char *url = args->url; + const int compressed = args->compressed; + { int current; ssize_t body_len = 0; ssize_t estimated_len = 0; smartlist_t *items = smartlist_new(); smartlist_t *dir_items = smartlist_new(); - int lifetime = 60; /* XXXX023 should actually use vote intervals. */ + int lifetime = 60; /* XXXX?? should actually use vote intervals. */ url += strlen("/tor/status-vote/"); current = !strcmpstart(url, "current/"); url = strchr(url, '/'); @@ -3136,8 +3132,18 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, smartlist_free(dir_items); goto done; } + done: + return 0; +} - if (!strcmpstart(url, "/tor/micro/d/")) { +/** Helper function for GET /tor/micro/d/... + */ +static int +handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args) +{ + const char *url = args->url; + const int compressed = args->compressed; + { smartlist_t *fps = smartlist_new(); dir_split_resource_into_fingerprints(url+strlen("/tor/micro/d/"), @@ -3150,7 +3156,7 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, smartlist_free(fps); goto done; } - dlen = dirserv_estimate_microdesc_size(fps, compressed); + size_t dlen = dirserv_estimate_microdesc_size(fps, compressed); if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) { log_info(LD_DIRSERV, "Client asked for server descriptors, but we've been " @@ -3173,12 +3179,24 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, goto done; } + done: + return 0; +} + +/** Helper function for GET /tor/{server,extra}/... + */ +static int +handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args) +{ + const char *url = args->url; + const int compressed = args->compressed; + const or_options_t *options = get_options(); if (!strcmpstart(url,"/tor/server/") || (!options->BridgeAuthoritativeDir && !options->BridgeRelay && !strcmpstart(url,"/tor/extra/"))) { + size_t dlen; int res; const char *msg; - const char *request_type = NULL; int cache_lifetime = 0; int is_extra = !strcmpstart(url,"/tor/extra/"); url += is_extra ? strlen("/tor/extra/") : strlen("/tor/server/"); @@ -3189,24 +3207,16 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, is_extra); if (!strcmpstart(url, "fp/")) { - request_type = compressed?"/tor/server/fp.z":"/tor/server/fp"; if (smartlist_len(conn->fingerprint_stack) == 1) cache_lifetime = ROUTERDESC_CACHE_LIFETIME; } else if (!strcmpstart(url, "authority")) { - request_type = compressed?"/tor/server/authority.z": - "/tor/server/authority"; cache_lifetime = ROUTERDESC_CACHE_LIFETIME; } else if (!strcmpstart(url, "all")) { - request_type = compressed?"/tor/server/all.z":"/tor/server/all"; cache_lifetime = FULL_DIR_CACHE_LIFETIME; } else if (!strcmpstart(url, "d/")) { - request_type = compressed?"/tor/server/d.z":"/tor/server/d"; if (smartlist_len(conn->fingerprint_stack) == 1) cache_lifetime = ROUTERDESC_BY_DIGEST_CACHE_LIFETIME; - } else { - request_type = "/tor/server/?"; } - (void) request_type; /* usable for note_request. */ if (!strcmpstart(url, "d/")) conn->dir_spool_src = is_extra ? DIR_SPOOL_EXTRA_BY_DIGEST : DIR_SPOOL_SERVER_BY_DIGEST; @@ -3242,8 +3252,19 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, } goto done; } + done: + return 0; +} - if (!strcmpstart(url,"/tor/keys/")) { +/** Helper function for GET /tor/keys/... + */ +static int +handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args) +{ + const char *url = args->url; + const int compressed = args->compressed; + const time_t if_modified_since = args->if_modified_since; + { smartlist_t *certs = smartlist_new(); ssize_t len = -1; if (!strcmp(url, "/tor/keys/all")) { @@ -3328,9 +3349,17 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, smartlist_free(certs); goto done; } + done: + return 0; +} - if (connection_dir_is_encrypted(conn) && - !strcmpstart(url,"/tor/rendezvous2/")) { +/** Helper function for GET /tor/rendezvous2/ + */ +static int +handle_get_rendezvous2(dir_connection_t *conn, const get_handler_args_t *args) +{ + const char *url = args->url; + if (connection_dir_is_encrypted(conn)) { /* Handle v2 rendezvous descriptor fetch request. */ const char *descp; const char *query = url + strlen("/tor/rendezvous2/"); @@ -3353,16 +3382,30 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, write_http_status_line(conn, 400, "Bad request"); } goto done; + } else { + /* Not encrypted! */ + write_http_status_line(conn, 404, "Not found"); } + done: + return 0; +} +/** Helper function for GET /tor/networkstatus-bridges + */ +static int +handle_get_networkstatus_bridges(dir_connection_t *conn, + const get_handler_args_t *args) +{ + const char *headers = args->headers; + + const or_options_t *options = get_options(); if (options->BridgeAuthoritativeDir && options->BridgePassword_AuthDigest_ && - connection_dir_is_encrypted(conn) && - !strcmp(url,"/tor/networkstatus-bridges")) { + connection_dir_is_encrypted(conn)) { char *status; char digest[DIGEST256_LEN]; - header = http_get_header(headers, "Authorization: Basic "); + char *header = http_get_header(headers, "Authorization: Basic "); if (header) crypto_digest256(digest, header, strlen(header), DIGEST_SHA256); @@ -3378,75 +3421,27 @@ directory_handle_command_get(dir_connection_t *conn, const char *headers, /* all happy now. send an answer. */ status = networkstatus_getinfo_by_purpose("bridge", time(NULL)); - dlen = strlen(status); + size_t dlen = strlen(status); write_http_response_header(conn, dlen, 0, 0); connection_write_to_buf(status, dlen, TO_CONN(conn)); tor_free(status); goto done; } + done: + return 0; +} - if (!strcmpstart(url,"/tor/bytes.txt")) { - char *bytes = directory_dump_request_log(); - size_t len = strlen(bytes); - write_http_response_header(conn, len, 0, 0); - connection_write_to_buf(bytes, len, TO_CONN(conn)); - tor_free(bytes); - goto done; - } - - if (!strcmp(url,"/tor/robots.txt")) { /* /robots.txt will have been - rewritten to /tor/robots.txt */ - char robots[] = "User-agent: *\r\nDisallow: /\r\n"; +/** Helper function for GET robots.txt or /tor/robots.txt */ +static int +handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args) +{ + (void)args; + { + const char robots[] = "User-agent: *\r\nDisallow: /\r\n"; size_t len = strlen(robots); write_http_response_header(conn, len, 0, ROBOTS_CACHE_LIFETIME); connection_write_to_buf(robots, len, TO_CONN(conn)); - goto done; } - -#if defined(EXPORTMALLINFO) && defined(HAVE_MALLOC_H) && defined(HAVE_MALLINFO) -#define ADD_MALLINFO_LINE(x) do { \ - smartlist_add_asprintf(lines, "%s %d\n", #x, mi.x); \ - }while(0); - - if (!strcmp(url,"/tor/mallinfo.txt") && - (tor_addr_eq_ipv4h(&conn->base_.addr, 0x7f000001ul))) { - char *result; - size_t len; - struct mallinfo mi; - smartlist_t *lines; - - memset(&mi, 0, sizeof(mi)); - mi = mallinfo(); - lines = smartlist_new(); - - ADD_MALLINFO_LINE(arena) - ADD_MALLINFO_LINE(ordblks) - ADD_MALLINFO_LINE(smblks) - ADD_MALLINFO_LINE(hblks) - ADD_MALLINFO_LINE(hblkhd) - ADD_MALLINFO_LINE(usmblks) - ADD_MALLINFO_LINE(fsmblks) - ADD_MALLINFO_LINE(uordblks) - ADD_MALLINFO_LINE(fordblks) - ADD_MALLINFO_LINE(keepcost) - - result = smartlist_join_strings(lines, "", 0, NULL); - SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp)); - smartlist_free(lines); - - len = strlen(result); - write_http_response_header(conn, len, 0, 0); - connection_write_to_buf(result, len, TO_CONN(conn)); - tor_free(result); - goto done; - } -#endif - - /* we didn't recognize the url */ - write_http_status_line(conn, 404, "Not found"); - - done: - tor_free(url_mem); return 0; } @@ -3773,17 +3768,83 @@ find_dl_schedule(download_status_t *dls, const or_options_t *options) return NULL; } -/* Find the current delay for dls based on schedule. - * Set dls->next_attempt_at based on now, and return the delay. +/** Decide which minimum and maximum delay step we want to use based on + * descriptor type in <b>dls</b> and <b>options</b>. + * Helper function for download_status_schedule_get_delay(). */ +STATIC void +find_dl_min_and_max_delay(download_status_t *dls, const or_options_t *options, + int *min, int *max) +{ + tor_assert(dls); + tor_assert(options); + tor_assert(min); + tor_assert(max); + + /* + * For now, just use the existing schedule config stuff and pick the + * first/last entries off to get min/max delay for backoff purposes + */ + const smartlist_t *schedule = find_dl_schedule(dls, options); + tor_assert(schedule != NULL && smartlist_len(schedule) >= 2); + *min = *((int *)(smartlist_get(schedule, 0))); + *max = *((int *)((smartlist_get(schedule, smartlist_len(schedule) - 1)))); +} + +/** Advance one delay step. The algorithm is to use the previous delay to + * compute an increment, we construct a value uniformly at random between + * delay and MAX(delay*2,delay+1). We then clamp that value to be no larger + * than max_delay, and return it. + * + * Requires that delay is less than INT_MAX, and delay is in [0,max_delay]. + */ +STATIC int +next_random_exponential_delay(int delay, int max_delay) +{ + /* Check preconditions */ + if (BUG(delay > max_delay)) + delay = max_delay; + if (BUG(delay == INT_MAX)) + delay -= 1; /* prevent overflow */ + if (BUG(delay < 0)) + delay = 0; + + /* How much are we willing to add to the delay? */ + int max_increment; + + if (delay) + max_increment = delay; /* no more than double. */ + else + max_increment = 1; /* we're always willing to slow down a little. */ + + /* the + 1 here is so that we include the end of the interval */ + int increment = crypto_rand_int(max_increment+1); + + if (increment < max_delay - delay) + return delay + increment; + else + return max_delay; +} + +/** Find the current delay for dls based on schedule or min_delay/ + * max_delay if we're using exponential backoff. If dls->backoff is + * DL_SCHED_RANDOM_EXPONENTIAL, we must have 0 <= min_delay <= max_delay <= + * INT_MAX, but schedule may be set to NULL; otherwise schedule is required. + * This function sets dls->next_attempt_at based on now, and returns the delay. * Helper for download_status_increment_failure and * download_status_increment_attempt. */ STATIC int download_status_schedule_get_delay(download_status_t *dls, const smartlist_t *schedule, + int min_delay, int max_delay, time_t now) { tor_assert(dls); - tor_assert(schedule); + /* We don't need a schedule if we're using random exponential backoff */ + tor_assert(dls->backoff == DL_SCHED_RANDOM_EXPONENTIAL || + schedule != NULL); + /* If we're using random exponential backoff, we do need min/max delay */ + tor_assert(dls->backoff != DL_SCHED_RANDOM_EXPONENTIAL || + (min_delay >= 0 && max_delay >= min_delay)); int delay = INT_MAX; uint8_t dls_schedule_position = (dls->increment_on @@ -3791,12 +3852,42 @@ download_status_schedule_get_delay(download_status_t *dls, ? dls->n_download_attempts : dls->n_download_failures); - if (dls_schedule_position < smartlist_len(schedule)) - delay = *(int *)smartlist_get(schedule, dls_schedule_position); - else if (dls_schedule_position == IMPOSSIBLE_TO_DOWNLOAD) - delay = INT_MAX; - else - delay = *(int *)smartlist_get(schedule, smartlist_len(schedule) - 1); + if (dls->backoff == DL_SCHED_DETERMINISTIC) { + if (dls_schedule_position < smartlist_len(schedule)) + delay = *(int *)smartlist_get(schedule, dls_schedule_position); + else if (dls_schedule_position == IMPOSSIBLE_TO_DOWNLOAD) + delay = INT_MAX; + else + delay = *(int *)smartlist_get(schedule, smartlist_len(schedule) - 1); + } else if (dls->backoff == DL_SCHED_RANDOM_EXPONENTIAL) { + /* Check if we missed a reset somehow */ + if (dls->last_backoff_position > dls_schedule_position) { + dls->last_backoff_position = 0; + dls->last_delay_used = 0; + } + + if (dls_schedule_position > 0) { + delay = dls->last_delay_used; + + while (dls->last_backoff_position < dls_schedule_position) { + /* Do one increment step */ + delay = next_random_exponential_delay(delay, max_delay); + /* Update our position */ + ++(dls->last_backoff_position); + } + } else { + /* If we're just starting out, use the minimum delay */ + delay = min_delay; + } + + /* Clamp it within min/max if we have them */ + if (min_delay >= 0 && delay < min_delay) delay = min_delay; + if (max_delay != INT_MAX && delay > max_delay) delay = max_delay; + + /* Store it for next time */ + dls->last_backoff_position = dls_schedule_position; + dls->last_delay_used = delay; + } /* A negative delay makes no sense. Knowing that delay is * non-negative allows us to safely do the wrapping check below. */ @@ -3857,6 +3948,8 @@ download_status_increment_failure(download_status_t *dls, int status_code, const char *item, int server, time_t now) { int increment = -1; + int min_delay = 0, max_delay = INT_MAX; + tor_assert(dls); /* only count the failure if it's permanent, or we're a server */ @@ -3877,7 +3970,9 @@ download_status_increment_failure(download_status_t *dls, int status_code, /* only return a failure retry time if this schedule increments on failures */ const smartlist_t *schedule = find_dl_schedule(dls, get_options()); - increment = download_status_schedule_get_delay(dls, schedule, now); + find_dl_min_and_max_delay(dls, get_options(), &min_delay, &max_delay); + increment = download_status_schedule_get_delay(dls, schedule, + min_delay, max_delay, now); } download_status_log_helper(item, !dls->increment_on, "failed", @@ -3906,6 +4001,8 @@ download_status_increment_attempt(download_status_t *dls, const char *item, time_t now) { int delay = -1; + int min_delay = 0, max_delay = INT_MAX; + tor_assert(dls); if (dls->increment_on == DL_SCHED_INCREMENT_FAILURE) { @@ -3920,7 +4017,9 @@ download_status_increment_attempt(download_status_t *dls, const char *item, ++dls->n_download_attempts; const smartlist_t *schedule = find_dl_schedule(dls, get_options()); - delay = download_status_schedule_get_delay(dls, schedule, now); + find_dl_min_and_max_delay(dls, get_options(), &min_delay, &max_delay); + delay = download_status_schedule_get_delay(dls, schedule, + min_delay, max_delay, now); download_status_log_helper(item, dls->increment_on, "attempted", "on failure", dls->n_download_attempts, @@ -3952,6 +4051,8 @@ download_status_reset(download_status_t *dls) dls->n_download_failures = 0; dls->n_download_attempts = 0; dls->next_attempt_at = time(NULL) + *(int *)smartlist_get(schedule, 0); + dls->last_backoff_position = 0; + dls->last_delay_used = 0; /* Don't reset dls->want_authority or dls->increment_on */ } @@ -4001,7 +4102,7 @@ dir_routerdesc_download_failed(smartlist_t *failed, int status_code, } SMARTLIST_FOREACH_BEGIN(failed, const char *, cp) { download_status_t *dls = NULL; - if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp)) < 0) { + if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp)) != DIGEST_LEN) { log_warn(LD_BUG, "Malformed fingerprint in list: %s", escaped(cp)); continue; } @@ -4098,9 +4199,10 @@ dir_split_resource_into_fingerprint_pairs(const char *res, "Skipping digest pair %s with missing dash.", escaped(cp)); } else { fp_pair_t pair; - if (base16_decode(pair.first, DIGEST_LEN, cp, HEX_DIGEST_LEN)<0 || - base16_decode(pair.second, - DIGEST_LEN, cp+HEX_DIGEST_LEN+1, HEX_DIGEST_LEN)<0) { + if (base16_decode(pair.first, DIGEST_LEN, + cp, HEX_DIGEST_LEN) != DIGEST_LEN || + base16_decode(pair.second,DIGEST_LEN, + cp+HEX_DIGEST_LEN+1, HEX_DIGEST_LEN) != DIGEST_LEN) { log_info(LD_DIR, "Skipping non-decodable digest pair %s", escaped(cp)); } else { smartlist_add(pairs_result, tor_memdup(&pair, sizeof(pair))); @@ -4178,8 +4280,9 @@ dir_split_resource_into_fingerprints(const char *resource, } d = tor_malloc_zero(digest_len); if (decode_hex ? - (base16_decode(d, digest_len, cp, hex_digest_len)<0) : - (base64_decode(d, digest_len, cp, base64_digest_len)<0)) { + (base16_decode(d, digest_len, cp, hex_digest_len) != digest_len) : + (base64_decode(d, digest_len, cp, base64_digest_len) + != digest_len)) { log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp)); smartlist_del_keeporder(fp_tmp, i--); goto again; diff --git a/src/or/directory.h b/src/or/directory.h index 7646cac03f..f04e7ab315 100644 --- a/src/or/directory.h +++ b/src/or/directory.h @@ -132,12 +132,12 @@ int download_status_get_n_failures(const download_status_t *dls); int download_status_get_n_attempts(const download_status_t *dls); time_t download_status_get_next_attempt_at(const download_status_t *dls); +int purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose); + #ifdef TOR_UNIT_TESTS /* Used only by directory.c and test_dir.c */ STATIC int parse_http_url(const char *headers, char **url); -STATIC int purpose_needs_anonymity(uint8_t dir_purpose, - uint8_t router_purpose); STATIC dirinfo_type_t dir_fetch_type(int dir_purpose, int router_purpose, const char *resource); STATIC int directory_handle_command_get(dir_connection_t *conn, @@ -146,6 +146,7 @@ STATIC int directory_handle_command_get(dir_connection_t *conn, size_t req_body_len); STATIC int download_status_schedule_get_delay(download_status_t *dls, const smartlist_t *schedule, + int min_delay, int max_delay, time_t now); STATIC char* authdir_type_to_string(dirinfo_type_t auth); @@ -154,6 +155,11 @@ STATIC int should_use_directory_guards(const or_options_t *options); STATIC zlib_compression_level_t choose_compression_level(ssize_t n_bytes); STATIC const smartlist_t *find_dl_schedule(download_status_t *dls, const or_options_t *options); +STATIC void find_dl_min_and_max_delay(download_status_t *dls, + const or_options_t *options, + int *min, int *max); +STATIC int next_random_exponential_delay(int delay, int max_delay); + #endif #endif diff --git a/src/or/dirserv.c b/src/or/dirserv.c index dafaed8bf2..64ebde6fdd 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -19,6 +19,7 @@ #include "dirvote.h" #include "hibernate.h" #include "keypin.h" +#include "main.h" #include "microdesc.h" #include "networkstatus.h" #include "nodelist.h" @@ -44,10 +45,6 @@ * directory authorities. */ #define MAX_UNTRUSTED_NETWORKSTATUSES 16 -extern time_t time_of_process_start; /* from main.c */ - -extern long stats_n_seconds_working; /* from main.c */ - /** Total number of routers with measured bandwidth; this is set by * dirserv_count_measured_bws() before the loop in * dirserv_generate_networkstatus_vote_obj() and checked by @@ -125,7 +122,8 @@ add_fingerprint_to_dir(const char *fp, authdir_config_t *list, fingerprint = tor_strdup(fp); tor_strstrip(fingerprint, " "); - if (base16_decode(d, DIGEST_LEN, fingerprint, strlen(fingerprint))) { + if (base16_decode(d, DIGEST_LEN, + fingerprint, strlen(fingerprint)) != DIGEST_LEN) { log_warn(LD_DIRSERV, "Couldn't decode fingerprint \"%s\"", escaped(fp)); tor_free(fingerprint); @@ -202,7 +200,7 @@ dirserv_load_fingerprint_file(void) tor_strstrip(fingerprint, " "); /* remove spaces */ if (strlen(fingerprint) != HEX_DIGEST_LEN || base16_decode(digest_tmp, sizeof(digest_tmp), - fingerprint, HEX_DIGEST_LEN) < 0) { + fingerprint, HEX_DIGEST_LEN) != sizeof(digest_tmp)) { log_notice(LD_CONFIG, "Invalid fingerprint (nickname '%s', " "fingerprint %s). Skipping.", @@ -349,7 +347,7 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, if (result & FP_REJECT) { if (msg) - *msg = "Fingerprint is marked rejected"; + *msg = "Fingerprint is marked rejected -- please contact us?"; return FP_REJECT; } else if (result & FP_INVALID) { if (msg) @@ -367,7 +365,7 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname, log_fn(severity, LD_DIRSERV, "Rejecting '%s' because of address '%s'", nickname, fmt_addr32(addr)); if (msg) - *msg = "Authdir is rejecting routers in this range."; + *msg = "Suspicious relay address range -- please contact us?"; return FP_REJECT; } if (!authdir_policy_valid_address(addr, or_port)) { @@ -823,7 +821,7 @@ running_long_enough_to_decide_unreachable(void) void dirserv_set_router_is_running(routerinfo_t *router, time_t now) { - /*XXXX024 This function is a mess. Separate out the part that calculates + /*XXXX This function is a mess. Separate out the part that calculates whether it's reachable and the part that tells rephist that the router was unreachable. */ @@ -985,94 +983,6 @@ router_is_active(const routerinfo_t *ri, const node_t *node, time_t now) return 1; } -/** Generate a new v1 directory and write it into a newly allocated string. - * Point *<b>dir_out</b> to the allocated string. Sign the - * directory with <b>private_key</b>. Return 0 on success, -1 on - * failure. If <b>complete</b> is set, give us all the descriptors; - * otherwise leave out non-running and non-valid ones. - */ -int -dirserv_dump_directory_to_string(char **dir_out, - crypto_pk_t *private_key) -{ - /* XXXX 024 Get rid of this function if we can confirm that nobody's - * fetching these any longer */ - char *cp; - char *identity_pkey; /* Identity key, DER64-encoded. */ - char *recommended_versions; - char digest[DIGEST_LEN]; - char published[ISO_TIME_LEN+1]; - char *buf = NULL; - size_t buf_len; - size_t identity_pkey_len; - time_t now = time(NULL); - - tor_assert(dir_out); - *dir_out = NULL; - - if (crypto_pk_write_public_key_to_string(private_key,&identity_pkey, - &identity_pkey_len)<0) { - log_warn(LD_BUG,"write identity_pkey to string failed!"); - return -1; - } - - recommended_versions = - format_versions_list(get_options()->RecommendedVersions); - - format_iso_time(published, now); - - buf_len = 2048+strlen(recommended_versions); - - buf = tor_malloc(buf_len); - /* We'll be comparing against buf_len throughout the rest of the - function, though strictly speaking we shouldn't be able to exceed - it. This is C, after all, so we may as well check for buffer - overruns.*/ - - tor_snprintf(buf, buf_len, - "signed-directory\n" - "published %s\n" - "recommended-software %s\n" - "router-status %s\n" - "dir-signing-key\n%s\n", - published, recommended_versions, "", - identity_pkey); - - tor_free(recommended_versions); - tor_free(identity_pkey); - - cp = buf + strlen(buf); - *cp = '\0'; - - /* These multiple strlcat calls are inefficient, but dwarfed by the RSA - signature. */ - if (strlcat(buf, "directory-signature ", buf_len) >= buf_len) - goto truncated; - if (strlcat(buf, get_options()->Nickname, buf_len) >= buf_len) - goto truncated; - if (strlcat(buf, "\n", buf_len) >= buf_len) - goto truncated; - - if (router_get_dir_hash(buf,digest)) { - log_warn(LD_BUG,"couldn't compute digest"); - tor_free(buf); - return -1; - } - note_crypto_pk_op(SIGN_DIR); - if (router_append_dirobj_signature(buf,buf_len,digest,DIGEST_LEN, - private_key)<0) { - tor_free(buf); - return -1; - } - - *dir_out = buf; - return 0; - truncated: - log_warn(LD_BUG,"tried to exceed string length."); - tor_free(buf); - return -1; -} - /********************************************************************/ /* A set of functions to answer questions about how we'd like to behave @@ -1329,7 +1239,7 @@ dirserv_thinks_router_is_unreliable(time_t now, { if (need_uptime) { if (!enough_mtbf_info) { - /* XXX024 Once most authorities are on v3, we should change the rule from + /* XXXX We should change the rule from * "use uptime if we don't have mtbf data" to "don't advertise Stable on * v3 if we don't have enough mtbf data." Or maybe not, since if we ever * hit a point where we need to reset a lot of authorities at once, @@ -2152,8 +2062,8 @@ routers_make_ed_keys_unique(smartlist_t *routers) const time_t ri2_pub = ri2->cache_info.published_on; if (ri2_pub < ri_pub || (ri2_pub == ri_pub && - memcmp(ri->cache_info.signed_descriptor_digest, - ri2->cache_info.signed_descriptor_digest,DIGEST_LEN)<0)) { + fast_memcmp(ri->cache_info.signed_descriptor_digest, + ri2->cache_info.signed_descriptor_digest,DIGEST_LEN)<0)) { digest256map_set(by_ed_key, pk, ri); ri2->omit_from_vote = 1; } else { @@ -2206,7 +2116,7 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs, rs->is_valid = node->is_valid; - if (node->is_fast && + if (node->is_fast && node->is_stable && ((options->AuthDirGuardBWGuarantee && routerbw_kb >= options->AuthDirGuardBWGuarantee/1000) || routerbw_kb >= MIN(guard_bandwidth_including_exits_kb, @@ -2365,7 +2275,8 @@ guardfraction_file_parse_guard_line(const char *guard_line, inputs_tmp = smartlist_get(sl, 0); if (strlen(inputs_tmp) != HEX_DIGEST_LEN || - base16_decode(guard_id, DIGEST_LEN, inputs_tmp, HEX_DIGEST_LEN)) { + base16_decode(guard_id, DIGEST_LEN, + inputs_tmp, HEX_DIGEST_LEN) != DIGEST_LEN) { tor_asprintf(err_msg, "bad digest '%s'", inputs_tmp); goto done; } @@ -2669,7 +2580,8 @@ measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line) cp+=strlen("node_id=$"); if (strlen(cp) != HEX_DIGEST_LEN || - base16_decode(out->node_id, DIGEST_LEN, cp, HEX_DIGEST_LEN)) { + base16_decode(out->node_id, DIGEST_LEN, + cp, HEX_DIGEST_LEN) != DIGEST_LEN) { log_warn(LD_DIRSERV, "Invalid node_id in bandwidth file line: %s", escaped(orig_line)); tor_free(line); @@ -3340,7 +3252,7 @@ lookup_cached_dir_by_fp(const char *fp) d = strmap_get(cached_consensuses, "ns"); } else if (memchr(fp, '\0', DIGEST_LEN) && cached_consensuses && (d = strmap_get(cached_consensuses, fp))) { - /* this here interface is a nasty hack XXXX024 */; + /* this here interface is a nasty hack XXXX */; } return d; } diff --git a/src/or/dirserv.h b/src/or/dirserv.h index 9a9725ad6f..3c914e9311 100644 --- a/src/or/dirserv.h +++ b/src/or/dirserv.h @@ -47,8 +47,6 @@ enum was_router_added_t dirserv_add_descriptor(routerinfo_t *ri, void dirserv_set_router_is_running(routerinfo_t *router, time_t now); int list_server_status_v1(smartlist_t *routers, char **router_status_out, int for_controller); -int dirserv_dump_directory_to_string(char **dir_out, - crypto_pk_t *private_key); char *dirserv_get_flag_thresholds_line(void); void dirserv_compute_bridge_flag_thresholds(void); diff --git a/src/or/dirvote.c b/src/or/dirvote.c index 62f85877fe..c31894a3b5 100644 --- a/src/or/dirvote.c +++ b/src/or/dirvote.c @@ -15,10 +15,12 @@ #include "policies.h" #include "rephist.h" #include "router.h" +#include "routerkeys.h" #include "routerlist.h" #include "routerparse.h" #include "entrynodes.h" /* needed for guardfraction methods */ #include "torcert.h" +#include "shared_random_state.h" /** * \file dirvote.c @@ -73,6 +75,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, char digest[DIGEST_LEN]; uint32_t addr; char *client_versions_line = NULL, *server_versions_line = NULL; + char *shared_random_vote_str = NULL; networkstatus_voter_info_t *voter; char *status = NULL; @@ -106,6 +109,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, SMARTLIST_FOREACH(v3_ns->package_lines, const char *, p, if (validate_recommended_package_line(p)) smartlist_add_asprintf(tmp, "package %s\n", p)); + smartlist_sort_strings(tmp); packages = smartlist_join_strings(tmp, "", 0, NULL); SMARTLIST_FOREACH(tmp, char *, cp, tor_free(cp)); smartlist_free(tmp); @@ -113,6 +117,9 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, packages = tor_strdup(""); } + /* Get shared random commitments/reveals line(s). */ + shared_random_vote_str = sr_get_string_for_vote(); + { char published[ISO_TIME_LEN+1]; char va[ISO_TIME_LEN+1]; @@ -152,7 +159,8 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, "flag-thresholds %s\n" "params %s\n" "dir-source %s %s %s %s %d %d\n" - "contact %s\n", + "contact %s\n" + "%s", /* shared randomness information */ v3_ns->type == NS_TYPE_VOTE ? "vote" : "opinion", methods, published, va, fu, vu, @@ -165,12 +173,15 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, params, voter->nickname, fingerprint, voter->address, fmt_addr32(addr), voter->dir_port, voter->or_port, - voter->contact); + voter->contact, + shared_random_vote_str ? + shared_random_vote_str : ""); tor_free(params); tor_free(flags); tor_free(flag_thresholds); tor_free(methods); + tor_free(shared_random_vote_str); if (!tor_digest_is_zero(voter->legacy_id_digest)) { char fpbuf[HEX_DIGEST_LEN+1]; @@ -607,15 +618,47 @@ compute_consensus_versions_list(smartlist_t *lst, int n_versioning) return result; } +/** Given a list of K=V values, return the int32_t value corresponding to + * KEYWORD=, or default_val if no such value exists, or if the value is + * corrupt. + */ +STATIC int32_t +dirvote_get_intermediate_param_value(const smartlist_t *param_list, + const char *keyword, + int32_t default_val) +{ + unsigned int n_found = 0; + int32_t value = default_val; + + SMARTLIST_FOREACH_BEGIN(param_list, const char *, k_v_pair) { + if (!strcmpstart(k_v_pair, keyword) && k_v_pair[strlen(keyword)] == '=') { + const char *integer_str = &k_v_pair[strlen(keyword)+1]; + int ok; + value = (int32_t) + tor_parse_long(integer_str, 10, INT32_MIN, INT32_MAX, &ok, NULL); + if (BUG(! ok)) + return default_val; + ++n_found; + } + } SMARTLIST_FOREACH_END(k_v_pair); + + if (n_found == 1) + return value; + else if (BUG(n_found > 1)) + return default_val; + else + return default_val; +} + /** Minimum number of directory authorities voting for a parameter to * include it in the consensus, if consensus method 12 or later is to be * used. See proposal 178 for details. */ #define MIN_VOTES_FOR_PARAM 3 -/** Helper: given a list of valid networkstatus_t, return a new string +/** Helper: given a list of valid networkstatus_t, return a new smartlist * containing the contents of the consensus network parameter set. */ -STATIC char * +STATIC smartlist_t * dirvote_compute_params(smartlist_t *votes, int method, int total_authorities) { int i; @@ -624,7 +667,6 @@ dirvote_compute_params(smartlist_t *votes, int method, int total_authorities) int cur_param_len; const char *cur_param; const char *eq; - char *result; const int n_votes = smartlist_len(votes); smartlist_t *output; @@ -646,8 +688,7 @@ dirvote_compute_params(smartlist_t *votes, int method, int total_authorities) if (smartlist_len(param_list) == 0) { tor_free(vals); - smartlist_free(param_list); - return NULL; + return param_list; } smartlist_sort_strings(param_list); @@ -695,12 +736,9 @@ dirvote_compute_params(smartlist_t *votes, int method, int total_authorities) } } SMARTLIST_FOREACH_END(param); - result = smartlist_join_strings(output, " ", 0, NULL); - SMARTLIST_FOREACH(output, char *, cp, tor_free(cp)); - smartlist_free(output); smartlist_free(param_list); tor_free(vals); - return result; + return output; } #define RANGE_CHECK(a,b,c,d,e,f,g,mx) \ @@ -1147,6 +1185,8 @@ networkstatus_compute_consensus(smartlist_t *votes, char *packages = NULL; int added_weights = 0; dircollator_t *collator = NULL; + smartlist_t *param_list = NULL; + tor_assert(flavor == FLAV_NS || flavor == FLAV_MICRODESC); tor_assert(total_authorities >= smartlist_len(votes)); tor_assert(total_authorities > 0); @@ -1291,14 +1331,31 @@ networkstatus_compute_consensus(smartlist_t *votes, tor_free(flaglist); } - params = dirvote_compute_params(votes, consensus_method, - total_authorities); - if (params) { + param_list = dirvote_compute_params(votes, consensus_method, + total_authorities); + if (smartlist_len(param_list)) { + params = smartlist_join_strings(param_list, " ", 0, NULL); smartlist_add(chunks, tor_strdup("params ")); smartlist_add(chunks, params); smartlist_add(chunks, tor_strdup("\n")); } + if (consensus_method >= MIN_METHOD_FOR_SHARED_RANDOM) { + int num_dirauth = get_n_authorities(V3_DIRINFO); + /* Default value of this is 2/3 of the total number of authorities. For + * instance, if we have 9 dirauth, the default value is 6. The following + * calculation will round it down. */ + int32_t num_srv_agreements = + dirvote_get_intermediate_param_value(param_list, + "AuthDirNumSRVAgreements", + (num_dirauth * 2) / 3); + /* Add the shared random value. */ + char *srv_lines = sr_get_string_for_consensus(votes, num_srv_agreements); + if (srv_lines != NULL) { + smartlist_add(chunks, srv_lines); + } + } + /* Sort the votes. */ smartlist_sort(votes, compare_votes_by_authority_id_); /* Add the authority sections. */ @@ -1350,7 +1407,7 @@ networkstatus_compute_consensus(smartlist_t *votes, if (consensus_method >= MIN_METHOD_TO_CLIP_UNMEASURED_BW) { char *max_unmeasured_param = NULL; - /* XXXX Extract this code into a common function */ + /* XXXX Extract this code into a common function. Or don't! see #19011 */ if (params) { if (strcmpstart(params, "maxunmeasuredbw=") == 0) max_unmeasured_param = params; @@ -1905,7 +1962,7 @@ networkstatus_compute_consensus(smartlist_t *votes, // Parse params, extract BW_WEIGHT_SCALE if present // DO NOT use consensus_param_bw_weight_scale() in this code! // The consensus is not formed yet! - /* XXXX Extract this code into a common function */ + /* XXXX Extract this code into a common function. Or not: #19011. */ if (params) { if (strcmpstart(params, "bwweightscale=") == 0) bw_weight_param = params; @@ -2025,6 +2082,8 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_free(flags); SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp)); smartlist_free(chunks); + SMARTLIST_FOREACH(param_list, char *, cp, tor_free(cp)); + smartlist_free(param_list); return result; } @@ -2168,7 +2227,7 @@ networkstatus_add_detached_signatures(networkstatus_t *target, } } if (!n_matches) { - *msg_out = "No regognized digests for given consensus flavor"; + *msg_out = "No recognized digests for given consensus flavor"; } } @@ -2510,50 +2569,60 @@ dirvote_get_start_of_next_interval(time_t now, int interval, int offset) return next; } -/** Scheduling information for a voting interval. */ -static struct { - /** When do we generate and distribute our vote for this interval? */ - time_t voting_starts; - /** When do we send an HTTP request for any votes that we haven't - * been posted yet?*/ - time_t fetch_missing_votes; - /** When do we give up on getting more votes and generate a consensus? */ - time_t voting_ends; - /** When do we send an HTTP request for any signatures we're expecting to - * see on the consensus? */ - time_t fetch_missing_signatures; - /** When do we publish the consensus? */ - time_t interval_starts; - - /* True iff we have generated and distributed our vote. */ - int have_voted; - /* True iff we've requested missing votes. */ - int have_fetched_missing_votes; - /* True iff we have built a consensus and sent the signatures around. */ - int have_built_consensus; - /* True iff we've fetched missing signatures. */ - int have_fetched_missing_signatures; - /* True iff we have published our consensus. */ - int have_published_consensus; -} voting_schedule = {0,0,0,0,0,0,0,0,0,0}; +/* Using the time <b>now</b>, return the next voting valid-after time. */ +time_t +get_next_valid_after_time(time_t now) +{ + time_t next_valid_after_time; + const or_options_t *options = get_options(); + voting_schedule_t *new_voting_schedule = + get_voting_schedule(options, now, LOG_INFO); + tor_assert(new_voting_schedule); + + next_valid_after_time = new_voting_schedule->interval_starts; + tor_free(new_voting_schedule); + + return next_valid_after_time; +} + +static voting_schedule_t voting_schedule; /** Set voting_schedule to hold the timing for the next vote we should be * doing. */ void dirvote_recalculate_timing(const or_options_t *options, time_t now) { + voting_schedule_t *new_voting_schedule; + + if (!authdir_mode_v3(options)) { + return; + } + + /* get the new voting schedule */ + new_voting_schedule = get_voting_schedule(options, now, LOG_NOTICE); + tor_assert(new_voting_schedule); + + /* Fill in the global static struct now */ + memcpy(&voting_schedule, new_voting_schedule, sizeof(voting_schedule)); + tor_free(new_voting_schedule); +} + +/* Populate and return a new voting_schedule_t that can be used to schedule + * voting. The object is allocated on the heap and it's the responsibility of + * the caller to free it. Can't fail. */ +voting_schedule_t * +get_voting_schedule(const or_options_t *options, time_t now, int severity) +{ int interval, vote_delay, dist_delay; time_t start; time_t end; networkstatus_t *consensus; + voting_schedule_t *new_voting_schedule; - if (!authdir_mode_v3(options)) - return; + new_voting_schedule = tor_malloc_zero(sizeof(voting_schedule_t)); consensus = networkstatus_get_live_consensus(now); - memset(&voting_schedule, 0, sizeof(voting_schedule)); - if (consensus) { interval = (int)( consensus->fresh_until - consensus->valid_after ); vote_delay = consensus->vote_seconds; @@ -2569,7 +2638,7 @@ dirvote_recalculate_timing(const or_options_t *options, time_t now) if (vote_delay + dist_delay > interval/2) vote_delay = dist_delay = interval / 4; - start = voting_schedule.interval_starts = + start = new_voting_schedule->interval_starts = dirvote_get_start_of_next_interval(now,interval, options->TestingV3AuthVotingStartOffset); end = dirvote_get_start_of_next_interval(start+1, interval, @@ -2577,18 +2646,21 @@ dirvote_recalculate_timing(const or_options_t *options, time_t now) tor_assert(end > start); - voting_schedule.fetch_missing_signatures = start - (dist_delay/2); - voting_schedule.voting_ends = start - dist_delay; - voting_schedule.fetch_missing_votes = start - dist_delay - (vote_delay/2); - voting_schedule.voting_starts = start - dist_delay - vote_delay; + new_voting_schedule->fetch_missing_signatures = start - (dist_delay/2); + new_voting_schedule->voting_ends = start - dist_delay; + new_voting_schedule->fetch_missing_votes = + start - dist_delay - (vote_delay/2); + new_voting_schedule->voting_starts = start - dist_delay - vote_delay; { char tbuf[ISO_TIME_LEN+1]; - format_iso_time(tbuf, voting_schedule.interval_starts); - log_notice(LD_DIR,"Choosing expected valid-after time as %s: " - "consensus_set=%d, interval=%d", - tbuf, consensus?1:0, interval); + format_iso_time(tbuf, new_voting_schedule->interval_starts); + tor_log(severity, LD_DIR,"Choosing expected valid-after time as %s: " + "consensus_set=%d, interval=%d", + tbuf, consensus?1:0, interval); } + + return new_voting_schedule; } /** Entry point: Take whatever voting actions are pending as of <b>now</b>. */ @@ -2637,6 +2709,9 @@ dirvote_act(const or_options_t *options, time_t now) dirvote_publish_consensus(); dirvote_clear_votes(0); voting_schedule.have_published_consensus = 1; + /* Update our shared random state with the consensus just published. */ + sr_act_post_consensus( + networkstatus_get_latest_consensus_by_flavor(FLAV_NS)); /* XXXX We will want to try again later if we haven't got enough * signatures yet. Implement this if it turns out to ever happen. */ dirvote_recalculate_timing(options, now); @@ -2916,7 +2991,8 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) /* Hey, it's a new cert! */ trusted_dirs_load_certs_from_string( vote->cert->cache_info.signed_descriptor_body, - TRUSTED_DIRS_CERTS_SRC_FROM_VOTE, 1 /*flush*/); + TRUSTED_DIRS_CERTS_SRC_FROM_VOTE, 1 /*flush*/, + NULL); if (!authority_cert_get_by_digests(vote->cert->cache_info.identity_digest, vote->cert->signing_key_digest)) { log_warn(LD_BUG, "We added a cert, but still couldn't find it."); @@ -2975,6 +3051,10 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) } } SMARTLIST_FOREACH_END(v); + /* This a valid vote, update our shared random state. */ + sr_handle_received_commits(vote->sr_info.commits, + vote->cert->identity_key); + pending_vote = tor_malloc_zero(sizeof(pending_vote_t)); pending_vote->vote_body = new_cached_dir(tor_strndup(vote_body, end_of_vote-vote_body), @@ -3019,6 +3099,30 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) return any_failed ? NULL : pending_vote; } +/* Write the votes in <b>pending_vote_list</b> to disk. */ +static void +write_v3_votes_to_disk(const smartlist_t *pending_vote_list) +{ + smartlist_t *votestrings = smartlist_new(); + char *votefile = NULL; + + SMARTLIST_FOREACH(pending_vote_list, pending_vote_t *, v, + { + sized_chunk_t *c = tor_malloc(sizeof(sized_chunk_t)); + c->bytes = v->vote_body->dir; + c->len = v->vote_body->dir_len; + smartlist_add(votestrings, c); /* collect strings to write to disk */ + }); + + votefile = get_datadir_fname("v3-status-votes"); + write_chunks_to_file(votefile, votestrings, 0, 0); + log_debug(LD_DIR, "Wrote votes to disk (%s)!", votefile); + + tor_free(votefile); + SMARTLIST_FOREACH(votestrings, sized_chunk_t *, c, tor_free(c)); + smartlist_free(votestrings); +} + /** Try to compute a v3 networkstatus consensus from the currently pending * votes. Return 0 on success, -1 on failure. Store the consensus in * pending_consensus: it won't be ready to be published until we have @@ -3028,8 +3132,8 @@ dirvote_compute_consensuses(void) { /* Have we got enough votes to try? */ int n_votes, n_voters, n_vote_running = 0; - smartlist_t *votes = NULL, *votestrings = NULL; - char *consensus_body = NULL, *signatures = NULL, *votefile; + smartlist_t *votes = NULL; + char *consensus_body = NULL, *signatures = NULL; networkstatus_t *consensus = NULL; authority_cert_t *my_cert; pending_consensus_t pending[N_CONSENSUS_FLAVORS]; @@ -3040,6 +3144,17 @@ dirvote_compute_consensuses(void) if (!pending_vote_list) pending_vote_list = smartlist_new(); + /* Write votes to disk */ + write_v3_votes_to_disk(pending_vote_list); + + /* Setup votes smartlist */ + votes = smartlist_new(); + SMARTLIST_FOREACH(pending_vote_list, pending_vote_t *, v, + { + smartlist_add(votes, v->vote); /* collect votes to compute consensus */ + }); + + /* See if consensus managed to achieve majority */ n_voters = get_n_authorities(V3_DIRINFO); n_votes = smartlist_len(pending_vote_list); if (n_votes <= n_voters/2) { @@ -3066,24 +3181,6 @@ dirvote_compute_consensuses(void) goto err; } - votes = smartlist_new(); - votestrings = smartlist_new(); - SMARTLIST_FOREACH(pending_vote_list, pending_vote_t *, v, - { - sized_chunk_t *c = tor_malloc(sizeof(sized_chunk_t)); - c->bytes = v->vote_body->dir; - c->len = v->vote_body->dir_len; - smartlist_add(votestrings, c); /* collect strings to write to disk */ - - smartlist_add(votes, v->vote); /* collect votes to compute consensus */ - }); - - votefile = get_datadir_fname("v3-status-votes"); - write_chunks_to_file(votefile, votestrings, 0, 0); - tor_free(votefile); - SMARTLIST_FOREACH(votestrings, sized_chunk_t *, c, tor_free(c)); - smartlist_free(votestrings); - { char legacy_dbuf[DIGEST_LEN]; crypto_pk_t *legacy_sign=NULL; @@ -3373,7 +3470,7 @@ dirvote_publish_consensus(void) continue; } - if (networkstatus_set_current_consensus(pending->body, name, 0)) + if (networkstatus_set_current_consensus(pending->body, name, 0, NULL)) log_warn(LD_DIR, "Error publishing %s consensus", name); else log_notice(LD_DIR, "Published %s consensus", name); @@ -3515,7 +3612,7 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method) if (consensus_method >= MIN_METHOD_FOR_P6_LINES && ri->ipv6_exit_policy) { - /* XXXX024 This doesn't match proposal 208, which says these should + /* XXXX+++ This doesn't match proposal 208, which says these should * be taken unchanged from the routerinfo. That's bogosity, IMO: * the proposal should have said to do this instead.*/ char *p6 = write_short_policy(ri->ipv6_exit_policy); diff --git a/src/or/dirvote.h b/src/or/dirvote.h index 0b1d284060..2a83802307 100644 --- a/src/or/dirvote.h +++ b/src/or/dirvote.h @@ -55,7 +55,7 @@ #define MIN_SUPPORTED_CONSENSUS_METHOD 13 /** The highest consensus method that we currently support. */ -#define MAX_SUPPORTED_CONSENSUS_METHOD 22 +#define MAX_SUPPORTED_CONSENSUS_METHOD 23 /** Lowest consensus method where microdesc consensuses omit any entry * with no microdesc. */ @@ -90,10 +90,15 @@ * ed25519 identities in microdescriptors. (Broken; see * consensus_method_is_supported() for more info.) */ #define MIN_METHOD_FOR_ED25519_ID_IN_MD 21 + /** Lowest consensus method where authorities vote on ed25519 ids and ensure * ed25519 id consistency. */ #define MIN_METHOD_FOR_ED25519_ID_VOTING 22 +/** Lowest consensus method where authorities may include a shared random + * value(s). */ +#define MIN_METHOD_FOR_SHARED_RANDOM 23 + /** Default bandwidth to clip unmeasured bandwidths to using method >= * MIN_METHOD_TO_CLIP_UNMEASURED_BW. (This is not a consensus method; do not * get confused with the above macros.) */ @@ -121,12 +126,44 @@ void ns_detached_signatures_free(ns_detached_signatures_t *s); authority_cert_t *authority_cert_dup(authority_cert_t *cert); /* vote scheduling */ + +/** Scheduling information for a voting interval. */ +typedef struct { + /** When do we generate and distribute our vote for this interval? */ + time_t voting_starts; + /** When do we send an HTTP request for any votes that we haven't + * been posted yet?*/ + time_t fetch_missing_votes; + /** When do we give up on getting more votes and generate a consensus? */ + time_t voting_ends; + /** When do we send an HTTP request for any signatures we're expecting to + * see on the consensus? */ + time_t fetch_missing_signatures; + /** When do we publish the consensus? */ + time_t interval_starts; + + /* True iff we have generated and distributed our vote. */ + int have_voted; + /* True iff we've requested missing votes. */ + int have_fetched_missing_votes; + /* True iff we have built a consensus and sent the signatures around. */ + int have_built_consensus; + /* True iff we've fetched missing signatures. */ + int have_fetched_missing_signatures; + /* True iff we have published our consensus. */ + int have_published_consensus; +} voting_schedule_t; + +voting_schedule_t *get_voting_schedule(const or_options_t *options, + time_t now, int severity); + void dirvote_get_preferred_voting_intervals(vote_timing_t *timing_out); time_t dirvote_get_start_of_next_interval(time_t now, int interval, int offset); void dirvote_recalculate_timing(const or_options_t *options, time_t now); void dirvote_act(const or_options_t *options, time_t now); +time_t get_next_valid_after_time(time_t now); /* invoked on timers and by outside triggers. */ struct pending_vote_t * dirvote_add_vote(const char *vote_body, @@ -173,9 +210,13 @@ document_signature_t *voter_get_sig_by_algorithm( digest_algorithm_t alg); #ifdef DIRVOTE_PRIVATE +STATIC int32_t dirvote_get_intermediate_param_value( + const smartlist_t *param_list, + const char *keyword, + int32_t default_val); STATIC char *format_networkstatus_vote(crypto_pk_t *private_key, networkstatus_t *v3_ns); -STATIC char *dirvote_compute_params(smartlist_t *votes, int method, +STATIC smartlist_t *dirvote_compute_params(smartlist_t *votes, int method, int total_authorities); STATIC char *compute_consensus_package_lines(smartlist_t *votes); STATIC char *make_consensus_method_list(int low, int high, const char *sep); diff --git a/src/or/dnsserv.c b/src/or/dnsserv.c index 74f17ce78c..edca50f6f9 100644 --- a/src/or/dnsserv.c +++ b/src/or/dnsserv.c @@ -130,7 +130,7 @@ evdns_server_callback(struct evdns_server_request *req, void *data_) tor_addr_copy(&TO_CONN(conn)->addr, &tor_addr); TO_CONN(conn)->port = port; - TO_CONN(conn)->address = tor_dup_addr(&tor_addr); + TO_CONN(conn)->address = tor_addr_to_str_dup(&tor_addr); if (q->type == EVDNS_TYPE_A || q->type == EVDNS_TYPE_AAAA || q->type == EVDNS_QTYPE_ALL) { @@ -205,7 +205,7 @@ dnsserv_launch_request(const char *name, int reverse, tor_addr_copy(&TO_CONN(conn)->addr, &control_conn->base_.addr); #ifdef AF_UNIX /* - * The control connection can be AF_UNIX and if so tor_dup_addr will + * The control connection can be AF_UNIX and if so tor_addr_to_str_dup will * unhelpfully say "<unknown address type>"; say "(Tor_internal)" * instead. */ @@ -214,11 +214,11 @@ dnsserv_launch_request(const char *name, int reverse, TO_CONN(conn)->address = tor_strdup("(Tor_internal)"); } else { TO_CONN(conn)->port = control_conn->base_.port; - TO_CONN(conn)->address = tor_dup_addr(&control_conn->base_.addr); + TO_CONN(conn)->address = tor_addr_to_str_dup(&control_conn->base_.addr); } #else TO_CONN(conn)->port = control_conn->base_.port; - TO_CONN(conn)->address = tor_dup_addr(&control_conn->base_.addr); + TO_CONN(conn)->address = tor_addr_to_str_dup(&control_conn->base_.addr); #endif if (reverse) diff --git a/src/or/entrynodes.c b/src/or/entrynodes.c index 310a948b35..6990bcd758 100644 --- a/src/or/entrynodes.c +++ b/src/or/entrynodes.c @@ -76,6 +76,14 @@ static const node_t *choose_random_entry_impl(cpath_build_state_t *state, int *n_options_out); static int num_bridges_usable(void); +/* Default number of entry guards in the case where the NumEntryGuards + * consensus parameter is not set */ +#define DEFAULT_N_GUARDS 1 +/* Minimum and maximum number of entry guards (in case the NumEntryGuards + * consensus parameter is set). */ +#define MIN_N_GUARDS 1 +#define MAX_N_GUARDS 10 + /** Return the list of entry guards, creating it if necessary. */ const smartlist_t * get_entry_guards(void) @@ -488,7 +496,8 @@ decide_num_guards(const or_options_t *options, int for_directory) return options->NumEntryGuards; /* Use the value from the consensus, or 3 if no guidance. */ - return networkstatus_get_param(NULL, "NumEntryGuards", 3, 1, 10); + return networkstatus_get_param(NULL, "NumEntryGuards", DEFAULT_N_GUARDS, + MIN_N_GUARDS, MAX_N_GUARDS); } /** If the use of entry guards is configured, choose more entry guards @@ -722,8 +731,9 @@ entry_guards_compute_status(const or_options_t *options, time_t now) * * If <b>mark_relay_status</b>, also call router_set_status() on this * relay. - * - * XXX024 change succeeded and mark_relay_status into 'int flags'. + */ +/* XXX We could change succeeded and mark_relay_status into 'int flags'. + * Too many boolean arguments is a recipe for confusion. */ int entry_guard_register_connect_status(const char *digest, int succeeded, @@ -1243,7 +1253,7 @@ entry_guards_parse_state(or_state_t *state, int set, char **msg) } else { strlcpy(node->nickname, smartlist_get(args,0), MAX_NICKNAME_LEN+1); if (base16_decode(node->identity, DIGEST_LEN, smartlist_get(args,1), - strlen(smartlist_get(args,1)))<0) { + strlen(smartlist_get(args,1))) != DIGEST_LEN) { *msg = tor_strdup("Unable to parse entry nodes: " "Bad hex digest for EntryGuard"); } @@ -1299,8 +1309,9 @@ entry_guards_parse_state(or_state_t *state, int set, char **msg) log_warn(LD_BUG, "EntryGuardAddedBy line is not long enough."); continue; } - if (base16_decode(d, sizeof(d), line->value, HEX_DIGEST_LEN)<0 || - line->value[HEX_DIGEST_LEN] != ' ') { + if (base16_decode(d, sizeof(d), + line->value, HEX_DIGEST_LEN) != sizeof(d) || + line->value[HEX_DIGEST_LEN] != ' ') { log_warn(LD_BUG, "EntryGuardAddedBy line %s does not begin with " "hex digest", escaped(line->value)); continue; @@ -1466,7 +1477,7 @@ entry_guards_parse_state(or_state_t *state, int set, char **msg) } entry_guards = new_entry_guards; entry_guards_dirty = 0; - /* XXX024 hand new_entry_guards to this func, and move it up a + /* XXX hand new_entry_guards to this func, and move it up a * few lines, so we don't have to re-dirty it */ if (remove_obsolete_entry_guards(now)) entry_guards_dirty = 1; @@ -2022,6 +2033,7 @@ bridge_add_from_config(bridge_line_t *bridge_line) if (bridge_line->transport_name) b->transport_name = bridge_line->transport_name; b->fetch_status.schedule = DL_SCHED_BRIDGE; + b->fetch_status.backoff = DL_SCHED_RANDOM_EXPONENTIAL; b->socks_args = bridge_line->socks_args; if (!bridge_list) bridge_list = smartlist_new(); @@ -2412,6 +2424,44 @@ num_bridges_usable(void) return n_options; } +/** Return a smartlist containing all bridge identity digests */ +MOCK_IMPL(smartlist_t *, +list_bridge_identities, (void)) +{ + smartlist_t *result = NULL; + char *digest_tmp; + + if (get_options()->UseBridges && bridge_list) { + result = smartlist_new(); + + SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, b) { + digest_tmp = tor_malloc(DIGEST_LEN); + memcpy(digest_tmp, b->identity, DIGEST_LEN); + smartlist_add(result, digest_tmp); + } SMARTLIST_FOREACH_END(b); + } + + return result; +} + +/** Get the download status for a bridge descriptor given its identity */ +MOCK_IMPL(download_status_t *, +get_bridge_dl_status_by_id, (const char *digest)) +{ + download_status_t *dl = NULL; + + if (digest && get_options()->UseBridges && bridge_list) { + SMARTLIST_FOREACH_BEGIN(bridge_list, bridge_info_t *, b) { + if (tor_memeq(digest, b->identity, DIGEST_LEN)) { + dl = &(b->fetch_status); + break; + } + } SMARTLIST_FOREACH_END(b); + } + + return dl; +} + /** Return 1 if we have at least one descriptor for an entry guard * (bridge or member of EntryNodes) and all descriptors we know are * down. Else return 0. If <b>act</b> is 1, then mark the down guards diff --git a/src/or/entrynodes.h b/src/or/entrynodes.h index 247c80940e..1021e67d43 100644 --- a/src/or/entrynodes.h +++ b/src/or/entrynodes.h @@ -179,5 +179,9 @@ guard_get_guardfraction_bandwidth(guardfraction_bandwidth_t *guardfraction_bw, int orig_bandwidth, uint32_t guardfraction_percentage); +MOCK_DECL(smartlist_t *, list_bridge_identities, (void)); +MOCK_DECL(download_status_t *, get_bridge_dl_status_by_id, + (const char *digest)); + #endif diff --git a/src/or/ext_orport.c b/src/or/ext_orport.c index aa1b3e26fe..8ba3c6afa3 100644 --- a/src/or/ext_orport.c +++ b/src/or/ext_orport.c @@ -461,8 +461,8 @@ connection_ext_or_handle_cmd_useraddr(connection_t *conn, return -1; { /* do some logging */ - char *old_address = tor_dup_addr(&conn->addr); - char *new_address = tor_dup_addr(&addr); + char *old_address = tor_addr_to_str_dup(&conn->addr); + char *new_address = tor_addr_to_str_dup(&addr); log_debug(LD_NET, "Received USERADDR." "We rewrite our address from '%s:%u' to '%s:%u'.", @@ -478,7 +478,7 @@ connection_ext_or_handle_cmd_useraddr(connection_t *conn, if (conn->address) { tor_free(conn->address); } - conn->address = tor_dup_addr(&addr); + conn->address = tor_addr_to_str_dup(&addr); return 0; } diff --git a/src/or/geoip.c b/src/or/geoip.c index b563db0418..874052495e 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -80,9 +80,9 @@ geoip_add_entry(const tor_addr_t *low, const tor_addr_t *high, intptr_t idx; void *idxplus1_; - if (tor_addr_family(low) != tor_addr_family(high)) + IF_BUG_ONCE(tor_addr_family(low) != tor_addr_family(high)) return; - if (tor_addr_compare(high, low, CMP_EXACT) < 0) + IF_BUG_ONCE(tor_addr_compare(high, low, CMP_EXACT) < 0) return; idxplus1_ = strmap_get_lc(country_idxplus1_by_lc_code, country); @@ -110,8 +110,8 @@ geoip_add_entry(const tor_addr_t *low, const tor_addr_t *high, smartlist_add(geoip_ipv4_entries, ent); } else if (tor_addr_family(low) == AF_INET6) { geoip_ipv6_entry_t *ent = tor_malloc_zero(sizeof(geoip_ipv6_entry_t)); - ent->ip_low = *tor_addr_to_in6(low); - ent->ip_high = *tor_addr_to_in6(high); + ent->ip_low = *tor_addr_to_in6_assert(low); + ent->ip_high = *tor_addr_to_in6_assert(high); ent->country = idx; smartlist_add(geoip_ipv6_entries, ent); } @@ -504,7 +504,7 @@ clientmap_entries_eq(const clientmap_entry_t *a, const clientmap_entry_t *b) } HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash, - clientmap_entries_eq); + clientmap_entries_eq) HT_GENERATE2(clientmap, clientmap_entry_t, node, clientmap_entry_hash, clientmap_entries_eq, 0.6, tor_reallocarray_, tor_free_) @@ -718,7 +718,7 @@ dirreq_map_ent_hash(const dirreq_map_entry_t *entry) } HT_PROTOTYPE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, - dirreq_map_ent_eq); + dirreq_map_ent_eq) HT_GENERATE2(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash, dirreq_map_ent_eq, 0.6, tor_reallocarray_, tor_free_) diff --git a/src/or/hibernate.c b/src/or/hibernate.c index 9408925d96..209aae01cf 100644 --- a/src/or/hibernate.c +++ b/src/or/hibernate.c @@ -28,13 +28,12 @@ hibernating, phase 2: #include "config.h" #include "connection.h" #include "connection_edge.h" +#include "control.h" #include "hibernate.h" #include "main.h" #include "router.h" #include "statefile.h" -extern long stats_n_seconds_working; /* published uptime */ - /** Are we currently awake, asleep, running out of bandwidth, or shutting * down? */ static hibernate_state_t hibernate_state = HIBERNATE_STATE_INITIAL; @@ -111,11 +110,34 @@ static int cfg_start_day = 0, cfg_start_min = 0; /** @} */ +static const char *hibernate_state_to_string(hibernate_state_t state); static void reset_accounting(time_t now); static int read_bandwidth_usage(void); static time_t start_of_accounting_period_after(time_t now); static time_t start_of_accounting_period_containing(time_t now); static void accounting_set_wakeup_time(void); +static void on_hibernate_state_change(hibernate_state_t prev_state); + +/** + * Return the human-readable name for the hibernation state <b>state</b> + */ +static const char * +hibernate_state_to_string(hibernate_state_t state) +{ + static char buf[64]; + switch (state) { + case HIBERNATE_STATE_EXITING: return "EXITING"; + case HIBERNATE_STATE_LOWBANDWIDTH: return "SOFT"; + case HIBERNATE_STATE_DORMANT: return "HARD"; + case HIBERNATE_STATE_INITIAL: + case HIBERNATE_STATE_LIVE: + return "AWAKE"; + default: + log_warn(LD_BUG, "unknown hibernate state %d", state); + tor_snprintf(buf, sizeof(buf), "unknown [%d]", state); + return buf; + } +} /* ************ * Functions for bandwidth accounting. @@ -935,6 +957,7 @@ consider_hibernation(time_t now) { int accounting_enabled = get_options()->AccountingMax != 0; char buf[ISO_TIME_LEN+1]; + hibernate_state_t prev_state = hibernate_state; /* If we're in 'exiting' mode, then we just shut down after the interval * elapses. */ @@ -990,6 +1013,10 @@ consider_hibernation(time_t now) hibernate_end_time_elapsed(now); } } + + /* Dispatch a controller event if the hibernation state changed. */ + if (hibernate_state != prev_state) + on_hibernate_state_change(prev_state); } /** Helper function: called when we get a GETINFO request for an @@ -1007,12 +1034,8 @@ getinfo_helper_accounting(control_connection_t *conn, if (!strcmp(question, "accounting/enabled")) { *answer = tor_strdup(accounting_is_enabled(get_options()) ? "1" : "0"); } else if (!strcmp(question, "accounting/hibernating")) { - if (hibernate_state == HIBERNATE_STATE_DORMANT) - *answer = tor_strdup("hard"); - else if (hibernate_state == HIBERNATE_STATE_LOWBANDWIDTH) - *answer = tor_strdup("soft"); - else - *answer = tor_strdup("awake"); + *answer = tor_strdup(hibernate_state_to_string(hibernate_state)); + tor_strlower(*answer); } else if (!strcmp(question, "accounting/bytes")) { tor_asprintf(answer, U64_FORMAT" "U64_FORMAT, U64_PRINTF_ARG(n_bytes_read_in_interval), @@ -1062,6 +1085,20 @@ getinfo_helper_accounting(control_connection_t *conn, return 0; } +/** + * Helper function: called when the hibernation state changes, and sends a + * SERVER_STATUS event to notify interested controllers of the accounting + * state change. + */ +static void +on_hibernate_state_change(hibernate_state_t prev_state) +{ + (void)prev_state; /* Should we do something with this? */ + control_event_server_status(LOG_NOTICE, + "HIBERNATION_STATUS STATUS=%s", + hibernate_state_to_string(hibernate_state)); +} + #ifdef TOR_UNIT_TESTS /** * Manually change the hibernation state. Private; used only by the unit diff --git a/src/or/include.am b/src/or/include.am index 712ae18406..744a507402 100644 --- a/src/or/include.am +++ b/src/or/include.am @@ -62,6 +62,8 @@ LIBTOR_A_SOURCES = \ src/or/onion.c \ src/or/onion_fast.c \ src/or/onion_tap.c \ + src/or/shared_random.c \ + src/or/shared_random_state.c \ src/or/transports.c \ src/or/periodic.c \ src/or/policies.c \ @@ -109,7 +111,7 @@ src_or_libtor_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) src_or_tor_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@ -src_or_tor_LDADD = src/or/libtor.a src/common/libor.a \ +src_or_tor_LDADD = src/or/libtor.a src/common/libor.a src/common/libor-ctime.a \ src/common/libor-crypto.a $(LIBKECCAK_TINY) $(LIBDONNA) \ src/common/libor-event.a src/trunnel/libor-trunnel.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ @@ -121,6 +123,7 @@ src_or_tor_cov_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_or_tor_cov_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) src_or_tor_cov_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@ src_or_tor_cov_LDADD = src/or/libtor-testing.a src/common/libor-testing.a \ + src/common/libor-ctime-testing.a \ src/common/libor-crypto-testing.a $(LIBKECCAK_TINY) $(LIBDONNA) \ src/common/libor-event-testing.a src/trunnel/libor-trunnel-testing.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ @@ -172,6 +175,8 @@ ORHEADERS = \ src/or/onion_ntor.h \ src/or/onion_tap.h \ src/or/or.h \ + src/or/shared_random.h \ + src/or/shared_random_state.h \ src/or/transports.h \ src/or/periodic.h \ src/or/policies.h \ diff --git a/src/or/keypin.c b/src/or/keypin.c index 1f82eccf86..335c793cd9 100644 --- a/src/or/keypin.c +++ b/src/or/keypin.c @@ -93,14 +93,14 @@ return (unsigned) siphash24g(a->ed25519_key, sizeof(a->ed25519_key)); } HT_PROTOTYPE(rsamap, keypin_ent_st, rsamap_node, keypin_ent_hash_rsa, - keypin_ents_eq_rsa); + keypin_ents_eq_rsa) HT_GENERATE2(rsamap, keypin_ent_st, rsamap_node, keypin_ent_hash_rsa, - keypin_ents_eq_rsa, 0.6, tor_reallocarray, tor_free_); + keypin_ents_eq_rsa, 0.6, tor_reallocarray, tor_free_) HT_PROTOTYPE(edmap, keypin_ent_st, edmap_node, keypin_ent_hash_ed, - keypin_ents_eq_ed); + keypin_ents_eq_ed) HT_GENERATE2(edmap, keypin_ent_st, edmap_node, keypin_ent_hash_ed, - keypin_ents_eq_ed, 0.6, tor_reallocarray, tor_free_); + keypin_ents_eq_ed, 0.6, tor_reallocarray, tor_free_) /** * Check whether we already have an entry in the key pinning table for a @@ -479,7 +479,7 @@ keypin_clear(void) HT_CLEAR(rsamap,&the_rsa_map); if (bad_entries) { - log_warn(LD_BUG, "Found %d discrepencies in the the keypin database.", + log_warn(LD_BUG, "Found %d discrepencies in the keypin database.", bad_entries); } } diff --git a/src/or/main.c b/src/or/main.c index 6b5619c7d6..4fc1498a98 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -57,6 +57,7 @@ #include "routerlist.h" #include "routerparse.h" #include "scheduler.h" +#include "shared_random.h" #include "statefile.h" #include "status.h" #include "util_process.h" @@ -962,7 +963,7 @@ conn_close_if_marked(int i) connection_stop_writing(conn); } if (connection_is_reading(conn)) { - /* XXXX024 We should make this code unreachable; if a connection is + /* XXXX+ We should make this code unreachable; if a connection is * marked for close and flushing, there is no point in reading to it * at all. Further, checking at this point is a bit of a hack: it * would make much more sense to react in @@ -1632,8 +1633,8 @@ rotate_x509_certificate_callback(time_t now, const or_options_t *options) * TLS context. */ log_info(LD_GENERAL,"Rotating tls context."); if (router_initialize_tls_context() < 0) { - log_warn(LD_BUG, "Error reinitializing TLS context"); - tor_assert(0); + log_err(LD_BUG, "Error reinitializing TLS context"); + tor_assert_unreached(); } /* We also make sure to rotate the TLS connections themselves if they've @@ -2447,6 +2448,13 @@ do_main_loop(void) cpu_init(); } + /* Setup shared random protocol subsystem. */ + if (authdir_mode_publishes_statuses(get_options())) { + if (sr_init(1) < 0) { + return -1; + } + } + /* set up once-a-second callback. */ if (! second_timer) { struct timeval one_second; @@ -2558,9 +2566,7 @@ run_main_loop_once(void) return -1; #endif } else { - if (ERRNO_IS_EINPROGRESS(e)) - log_warn(LD_BUG, - "libevent call returned EINPROGRESS? Please report."); + tor_assert_nonfatal_once(! ERRNO_IS_EINPROGRESS(e)); log_debug(LD_NET,"libevent call interrupted."); /* You can't trust the results of this poll(). Go back to the * top of the big for loop. */ @@ -2691,9 +2697,6 @@ get_uptime,(void)) return stats_n_seconds_working; } -extern uint64_t rephist_total_alloc; -extern uint32_t rephist_total_num; - /** * Write current memory usage information to the log. */ @@ -3050,6 +3053,9 @@ tor_init(int argc, char *argv[]) log_warn(LD_NET, "Problem initializing libevent RNG."); } + /* Scan/clean unparseable descroptors; after reading config */ + routerparse_init(); + return 0; } @@ -3151,6 +3157,7 @@ tor_free_all(int postfork) scheduler_free_all(); nodelist_free_all(); microdesc_free_all(); + routerparse_free_all(); ext_orport_free_all(); control_free_all(); sandbox_free_getaddrinfo_cache(); @@ -3215,6 +3222,9 @@ tor_cleanup(void) accounting_record_bandwidth_usage(now, get_or_state()); or_state_mark_dirty(get_or_state(), 0); /* force an immediate save. */ or_state_save(now); + if (authdir_mode(options)) { + sr_save_and_cleanup(); + } if (authdir_mode_tests_reachability(options)) rep_hist_record_mtbf_data(now, 0); keypin_close_journal(); @@ -3373,6 +3383,7 @@ sandbox_init_filter(void) OPEN_DATADIR_SUFFIX("cached-extrainfo.new", ".tmp"); OPEN_DATADIR("cached-extrainfo.tmp.tmp"); OPEN_DATADIR_SUFFIX("state", ".tmp"); + OPEN_DATADIR_SUFFIX("sr-state", ".tmp"); OPEN_DATADIR_SUFFIX("unparseable-desc", ".tmp"); OPEN_DATADIR_SUFFIX("v3-status-votes", ".tmp"); OPEN_DATADIR("key-pinning-journal"); @@ -3425,6 +3436,7 @@ sandbox_init_filter(void) RENAME_SUFFIX("cached-extrainfo", ".new"); RENAME_SUFFIX("cached-extrainfo.new", ".tmp"); RENAME_SUFFIX("state", ".tmp"); + RENAME_SUFFIX("sr-state", ".tmp"); RENAME_SUFFIX("unparseable-desc", ".tmp"); RENAME_SUFFIX("v3-status-votes", ".tmp"); diff --git a/src/or/main.h b/src/or/main.h index ad865b8124..31a22de424 100644 --- a/src/or/main.h +++ b/src/or/main.h @@ -75,6 +75,14 @@ int tor_main(int argc, char *argv[]); int do_main_loop(void); int tor_init(int argc, char **argv); +extern time_t time_of_process_start; +extern long stats_n_seconds_working; +extern int quiet_level; +extern int global_read_bucket; +extern int global_write_bucket; +extern int global_relayed_read_bucket; +extern int global_relayed_write_bucket; + #ifdef MAIN_PRIVATE STATIC void init_connection_lists(void); STATIC void close_closeable_connections(void); diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 5b5c29a6d2..130259a29f 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -69,7 +69,7 @@ microdesc_eq_(microdesc_t *a, microdesc_t *b) } HT_PROTOTYPE(microdesc_map, microdesc_t, node, - microdesc_hash_, microdesc_eq_); + microdesc_hash_, microdesc_eq_) HT_GENERATE2(microdesc_map, microdesc_t, node, microdesc_hash_, microdesc_eq_, 0.6, tor_reallocarray_, tor_free_) @@ -925,7 +925,7 @@ we_use_microdescriptors_for_circuits(const or_options_t *options) return 0; /* Otherwise, we decide that we'll use microdescriptors iff we are * not a server, and we're not autofetching everything. */ - /* XXX023 what does not being a server have to do with it? also there's + /* XXXX++ what does not being a server have to do with it? also there's * a partitioning issue here where bridges differ from clients. */ ret = !server_mode(options) && !options->FetchUselessDescriptors; } diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 51fc01108f..0dfb8afcce 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -32,7 +32,9 @@ #include "router.h" #include "routerlist.h" #include "routerparse.h" +#include "shared_random.h" #include "transports.h" +#include "torcert.h" /** Map from lowercase nickname to identity digest of named server, if any. */ static strmap_t *named_server_map = NULL; @@ -86,9 +88,9 @@ static time_t time_to_download_next_consensus[N_CONSENSUS_FLAVORS]; static download_status_t consensus_dl_status[N_CONSENSUS_FLAVORS] = { { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_ANY_DIRSERVER, - DL_SCHED_INCREMENT_FAILURE }, + DL_SCHED_INCREMENT_FAILURE, DL_SCHED_RANDOM_EXPONENTIAL, 0, 0 }, { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_ANY_DIRSERVER, - DL_SCHED_INCREMENT_FAILURE }, + DL_SCHED_INCREMENT_FAILURE, DL_SCHED_RANDOM_EXPONENTIAL, 0, 0 }, }; #define N_CONSENSUS_BOOTSTRAP_SCHEDULES 2 @@ -105,10 +107,10 @@ static download_status_t consensus_bootstrap_dl_status[N_CONSENSUS_BOOTSTRAP_SCHEDULES] = { { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_AUTHORITY, - DL_SCHED_INCREMENT_ATTEMPT }, + DL_SCHED_INCREMENT_ATTEMPT, DL_SCHED_RANDOM_EXPONENTIAL, 0, 0 }, /* During bootstrap, DL_WANT_ANY_DIRSERVER means "use fallbacks". */ { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_ANY_DIRSERVER, - DL_SCHED_INCREMENT_ATTEMPT }, + DL_SCHED_INCREMENT_ATTEMPT, DL_SCHED_RANDOM_EXPONENTIAL, 0, 0 }, }; /** True iff we have logged a warning about this OR's version being older than @@ -173,7 +175,7 @@ router_reload_consensus_networkstatus(void) } s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); if (s) { - if (networkstatus_set_current_consensus(s, flavor, flags) < -1) { + if (networkstatus_set_current_consensus(s, flavor, flags, NULL) < -1) { log_warn(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"", flavor, filename); } @@ -191,7 +193,8 @@ router_reload_consensus_networkstatus(void) s = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL); if (s) { if (networkstatus_set_current_consensus(s, flavor, - flags|NSSET_WAS_WAITING_FOR_CERTS)) { + flags|NSSET_WAS_WAITING_FOR_CERTS, + NULL)) { log_info(LD_FS, "Couldn't load consensus %s networkstatus from \"%s\"", flavor, filename); } @@ -319,6 +322,14 @@ networkstatus_vote_free(networkstatus_t *ns) digestmap_free(ns->desc_digest_map, NULL); + if (ns->sr_info.commits) { + SMARTLIST_FOREACH(ns->sr_info.commits, sr_commit_t *, c, + sr_commit_free(c)); + smartlist_free(ns->sr_info.commits); + } + tor_free(ns->sr_info.previous_srv); + tor_free(ns->sr_info.current_srv); + memwipe(ns, 11, sizeof(*ns)); tor_free(ns); } @@ -658,6 +669,43 @@ router_get_consensus_status_by_descriptor_digest(networkstatus_t *consensus, consensus, digest); } +/** Return a smartlist of all router descriptor digests in a consensus */ +static smartlist_t * +router_get_descriptor_digests_in_consensus(networkstatus_t *consensus) +{ + smartlist_t *result = smartlist_new(); + digestmap_iter_t *i; + const char *digest; + void *rs; + char *digest_tmp; + + for (i = digestmap_iter_init(consensus->desc_digest_map); + !(digestmap_iter_done(i)); + i = digestmap_iter_next(consensus->desc_digest_map, i)) { + digestmap_iter_get(i, &digest, &rs); + digest_tmp = tor_malloc(DIGEST_LEN); + memcpy(digest_tmp, digest, DIGEST_LEN); + smartlist_add(result, digest_tmp); + } + + return result; +} + +/** Return a smartlist of all router descriptor digests in the current + * consensus */ +MOCK_IMPL(smartlist_t *, +router_get_descriptor_digests,(void)) +{ + smartlist_t *result = NULL; + + if (current_ns_consensus) { + result = + router_get_descriptor_digests_in_consensus(current_ns_consensus); + } + + return result; +} + /** Given the digest of a router descriptor, return its current download * status, or NULL if the digest is unrecognized. */ MOCK_IMPL(download_status_t *, @@ -1160,13 +1208,13 @@ update_certificate_downloads(time_t now) for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { if (consensus_waiting_for_certs[i].consensus) authority_certs_fetch_missing(consensus_waiting_for_certs[i].consensus, - now); + now, NULL); } if (current_ns_consensus) - authority_certs_fetch_missing(current_ns_consensus, now); + authority_certs_fetch_missing(current_ns_consensus, now, NULL); if (current_md_consensus) - authority_certs_fetch_missing(current_md_consensus, now); + authority_certs_fetch_missing(current_md_consensus, now, NULL); } /** Return 1 if we have a consensus but we don't have enough certificates @@ -1178,10 +1226,56 @@ consensus_is_waiting_for_certs(void) ? 1 : 0; } +/** Look up the currently active (depending on bootstrap status) download + * status for this consensus flavor and return a pointer to it. + */ +MOCK_IMPL(download_status_t *, +networkstatus_get_dl_status_by_flavor,(consensus_flavor_t flavor)) +{ + download_status_t *dl = NULL; + const int we_are_bootstrapping = + networkstatus_consensus_is_bootstrapping(time(NULL)); + + if ((int)flavor <= N_CONSENSUS_FLAVORS) { + dl = &((we_are_bootstrapping ? + consensus_bootstrap_dl_status : consensus_dl_status)[flavor]); + } + + return dl; +} + +/** Look up the bootstrap download status for this consensus flavor + * and return a pointer to it. */ +MOCK_IMPL(download_status_t *, +networkstatus_get_dl_status_by_flavor_bootstrap,(consensus_flavor_t flavor)) +{ + download_status_t *dl = NULL; + + if ((int)flavor <= N_CONSENSUS_FLAVORS) { + dl = &(consensus_bootstrap_dl_status[flavor]); + } + + return dl; +} + +/** Look up the running (non-bootstrap) download status for this consensus + * flavor and return a pointer to it. */ +MOCK_IMPL(download_status_t *, +networkstatus_get_dl_status_by_flavor_running,(consensus_flavor_t flavor)) +{ + download_status_t *dl = NULL; + + if ((int)flavor <= N_CONSENSUS_FLAVORS) { + dl = &(consensus_dl_status[flavor]); + } + + return dl; +} + /** Return the most recent consensus that we have downloaded, or NULL if we * don't have one. */ -networkstatus_t * -networkstatus_get_latest_consensus(void) +MOCK_IMPL(networkstatus_t *, +networkstatus_get_latest_consensus,(void)) { return current_consensus; } @@ -1203,8 +1297,8 @@ networkstatus_get_latest_consensus_by_flavor,(consensus_flavor_t f)) /** Return the most recent consensus that we have downloaded, or NULL if it is * no longer live. */ -networkstatus_t * -networkstatus_get_live_consensus(time_t now) +MOCK_IMPL(networkstatus_t *, +networkstatus_get_live_consensus,(time_t now)) { if (current_consensus && current_consensus->valid_after <= now && @@ -1460,6 +1554,10 @@ networkstatus_set_current_consensus_from_ns(networkstatus_t *c, * If flags & NSSET_ACCEPT_OBSOLETE, then we should be willing to take this * consensus, even if it comes from many days in the past. * + * If source_dir is non-NULL, it's the identity digest for a directory that + * we've just successfully retrieved a consensus or certificates from, so try + * it first to fetch any missing certificates. + * * Return 0 on success, <0 on failure. On failure, caller should increment * the failure count as appropriate. * @@ -1469,7 +1567,8 @@ networkstatus_set_current_consensus_from_ns(networkstatus_t *c, int networkstatus_set_current_consensus(const char *consensus, const char *flavor, - unsigned flags) + unsigned flags, + const char *source_dir) { networkstatus_t *c=NULL; int r, result = -1; @@ -1591,7 +1690,7 @@ networkstatus_set_current_consensus(const char *consensus, write_str_to_file(unverified_fname, consensus, 0); } if (dl_certs) - authority_certs_fetch_missing(c, now); + authority_certs_fetch_missing(c, now, source_dir); /* This case is not a success or a failure until we get the certs * or fail to get the certs. */ result = 0; @@ -1629,7 +1728,7 @@ networkstatus_set_current_consensus(const char *consensus, /* Are we missing any certificates at all? */ if (r != 1 && dl_certs) - authority_certs_fetch_missing(c, now); + authority_certs_fetch_missing(c, now, source_dir); if (flav == usable_consensus_flavor()) { notify_control_networkstatus_changed(current_consensus, c); @@ -1703,7 +1802,7 @@ networkstatus_set_current_consensus(const char *consensus, channel_set_cmux_policy_everywhere(NULL); } - /* XXXX024 this call might be unnecessary here: can changing the + /* XXXX this call might be unnecessary here: can changing the * current consensus really alter our view of any OR's rate limits? */ connection_or_update_token_buckets(get_connection_array(), options); @@ -1752,9 +1851,14 @@ networkstatus_set_current_consensus(const char *consensus, } /** Called when we have gotten more certificates: see whether we can - * now verify a pending consensus. */ + * now verify a pending consensus. + * + * If source_dir is non-NULL, it's the identity digest for a directory that + * we've just successfully retrieved certificates from, so try it first to + * fetch any missing certificates. + */ void -networkstatus_note_certs_arrived(void) +networkstatus_note_certs_arrived(const char *source_dir) { int i; for (i=0; i<N_CONSENSUS_FLAVORS; ++i) { @@ -1766,7 +1870,8 @@ networkstatus_note_certs_arrived(void) if (!networkstatus_set_current_consensus( waiting_body, networkstatus_get_flavor_name(i), - NSSET_WAS_WAITING_FOR_CERTS)) { + NSSET_WAS_WAITING_FOR_CERTS, + source_dir)) { tor_free(waiting_body); } } @@ -2204,7 +2309,7 @@ getinfo_helper_networkstatus(control_connection_t *conn, if (*q == '$') ++q; - if (base16_decode(d, DIGEST_LEN, q, strlen(q))) { + if (base16_decode(d, DIGEST_LEN, q, strlen(q)) != DIGEST_LEN) { *errmsg = "Data not decodeable as hex"; return -1; } diff --git a/src/or/networkstatus.h b/src/or/networkstatus.h index ac93e5de91..71f36b69ed 100644 --- a/src/or/networkstatus.h +++ b/src/or/networkstatus.h @@ -38,6 +38,17 @@ routerstatus_t *networkstatus_vote_find_mutable_entry(networkstatus_t *ns, int networkstatus_vote_find_entry_idx(networkstatus_t *ns, const char *digest, int *found_out); +MOCK_DECL(download_status_t *, + networkstatus_get_dl_status_by_flavor, + (consensus_flavor_t flavor)); +MOCK_DECL(download_status_t *, + networkstatus_get_dl_status_by_flavor_bootstrap, + (consensus_flavor_t flavor)); +MOCK_DECL(download_status_t *, + networkstatus_get_dl_status_by_flavor_running, + (consensus_flavor_t flavor)); + +MOCK_DECL(smartlist_t *, router_get_descriptor_digests, (void)); MOCK_DECL(download_status_t *,router_get_dl_status_by_descriptor_digest, (const char *d)); @@ -64,10 +75,10 @@ void update_certificate_downloads(time_t now); int consensus_is_waiting_for_certs(void); int client_would_use_router(const routerstatus_t *rs, time_t now, const or_options_t *options); -networkstatus_t *networkstatus_get_latest_consensus(void); +MOCK_DECL(networkstatus_t *,networkstatus_get_latest_consensus,(void)); MOCK_DECL(networkstatus_t *,networkstatus_get_latest_consensus_by_flavor, (consensus_flavor_t f)); -networkstatus_t *networkstatus_get_live_consensus(time_t now); +MOCK_DECL(networkstatus_t *, networkstatus_get_live_consensus,(time_t now)); networkstatus_t *networkstatus_get_reasonably_live_consensus(time_t now, int flavor); MOCK_DECL(int, networkstatus_consensus_is_bootstrapping,(time_t now)); @@ -84,8 +95,9 @@ int networkstatus_consensus_is_already_downloading(const char *resource); #define NSSET_REQUIRE_FLAVOR 16 int networkstatus_set_current_consensus(const char *consensus, const char *flavor, - unsigned flags); -void networkstatus_note_certs_arrived(void); + unsigned flags, + const char *source_dir); +void networkstatus_note_certs_arrived(const char *source_dir); void routers_update_all_from_networkstatus(time_t now, int dir_version); void routers_update_status_from_consensus_networkstatus(smartlist_t *routers, int reset_failures); diff --git a/src/or/nodelist.c b/src/or/nodelist.c index 89b5355c8d..a49bf03f61 100644 --- a/src/or/nodelist.c +++ b/src/or/nodelist.c @@ -77,7 +77,7 @@ node_id_eq(const node_t *node1, const node_t *node2) return tor_memeq(node1->identity, node2->identity, DIGEST_LEN); } -HT_PROTOTYPE(nodelist_map, node_t, ht_ent, node_id_hash, node_id_eq); +HT_PROTOTYPE(nodelist_map, node_t, ht_ent, node_id_hash, node_id_eq) HT_GENERATE2(nodelist_map, node_t, ht_ent, node_id_hash, node_id_eq, 0.6, tor_reallocarray_, tor_free_) diff --git a/src/or/onion.c b/src/or/onion.c index d6ef3673dd..7c7f97fc42 100644 --- a/src/or/onion.c +++ b/src/or/onion.c @@ -38,9 +38,9 @@ typedef struct onion_queue_t { /** Array of queues of circuits waiting for CPU workers. An element is NULL * if that queue is empty.*/ -TOR_TAILQ_HEAD(onion_queue_head_t, onion_queue_t) - ol_list[MAX_ONION_HANDSHAKE_TYPE+1] = { - TOR_TAILQ_HEAD_INITIALIZER(ol_list[0]), /* tap */ +static TOR_TAILQ_HEAD(onion_queue_head_t, onion_queue_t) + ol_list[MAX_ONION_HANDSHAKE_TYPE+1] = +{ TOR_TAILQ_HEAD_INITIALIZER(ol_list[0]), /* tap */ TOR_TAILQ_HEAD_INITIALIZER(ol_list[1]), /* fast */ TOR_TAILQ_HEAD_INITIALIZER(ol_list[2]), /* ntor */ }; @@ -51,7 +51,7 @@ static int ol_entries[MAX_ONION_HANDSHAKE_TYPE+1]; static int num_ntors_per_tap(void); static void onion_queue_entry_remove(onion_queue_t *victim); -/* XXXX024 Check lengths vs MAX_ONIONSKIN_{CHALLENGE,REPLY}_LEN. +/* XXXX Check lengths vs MAX_ONIONSKIN_{CHALLENGE,REPLY}_LEN. * * (By which I think I meant, "make sure that no * X_ONIONSKIN_CHALLENGE/REPLY_LEN is greater than @@ -527,7 +527,7 @@ onion_skin_server_handshake(int type, * <b>rend_authenticator_out</b> to the "KH" field that can be used to * establish introduction points at this hop, and return 0. On failure, * return -1, and set *msg_out to an error message if this is worth - * complaining to the usre about. */ + * complaining to the user about. */ int onion_skin_client_handshake(int type, const onion_handshake_state_t *handshake_state, diff --git a/src/or/onion_ntor.c b/src/or/onion_ntor.c index 9f97a4cfbe..33afc27895 100644 --- a/src/or/onion_ntor.c +++ b/src/or/onion_ntor.c @@ -47,7 +47,7 @@ typedef struct tweakset_t { } tweakset_t; /** The tweaks to be used with our handshake. */ -const tweakset_t proto1_tweaks = { +static const tweakset_t proto1_tweaks = { #define PROTOID "ntor-curve25519-sha256-1" #define PROTOID_LEN 24 PROTOID ":mac", diff --git a/src/or/or.h b/src/or/or.h index 2252f38161..ed799b98ff 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -14,14 +14,6 @@ #include "orconfig.h" -#if defined(__clang_analyzer__) || defined(__COVERITY__) -/* If we're building for a static analysis, turn on all the off-by-default - * features. */ -#ifndef INSTRUMENT_DOWNLOADS -#define INSTRUMENT_DOWNLOADS 1 -#endif -#endif - #ifdef HAVE_UNISTD_H #include <unistd.h> #endif @@ -784,7 +776,7 @@ typedef enum rend_auth_type_t { /** Client-side configuration of authorization for a hidden service. */ typedef struct rend_service_authorization_t { - char descriptor_cookie[REND_DESC_COOKIE_LEN]; + uint8_t descriptor_cookie[REND_DESC_COOKIE_LEN]; char onion_address[REND_SERVICE_ADDRESS_LEN+1]; rend_auth_type_t auth_type; } rend_service_authorization_t; @@ -1294,21 +1286,26 @@ typedef struct connection_t { time_t timestamp_created; /**< When was this connection_t created? */ - /* XXXX_IP6 make this IPv6-capable */ int socket_family; /**< Address family of this connection's socket. Usually - * AF_INET, but it can also be AF_UNIX, or in the future - * AF_INET6 */ - tor_addr_t addr; /**< IP of the other side of the connection; used to - * identify routers, along with port. */ - uint16_t port; /**< If non-zero, port on the other end - * of the connection. */ + * AF_INET, but it can also be AF_UNIX, or AF_INET6 */ + tor_addr_t addr; /**< IP that socket "s" is directly connected to; + * may be the IP address for a proxy or pluggable transport, + * see "address" for the address of the final destination. + */ + uint16_t port; /**< If non-zero, port that socket "s" is directly connected + * to; may be the port for a proxy or pluggable transport, + * see "address" for the port at the final destination. */ uint16_t marked_for_close; /**< Should we close this conn on the next * iteration of the main loop? (If true, holds * the line number where this connection was * marked.) */ const char *marked_for_close_file; /**< For debugging: in which file were * we marked for close? */ - char *address; /**< FQDN (or IP) of the other end. + char *address; /**< FQDN (or IP) and port of the final destination for this + * connection; this is always the remote address, it is + * passed to a proxy or pluggable transport if one in use. + * See "addr" and "port" for the address that socket "s" is + * directly connected to. * strdup into this, because free_connection() frees it. */ /** Another connection that's connected to this one in lieu of a socket. */ struct connection_t *linked_conn; @@ -1990,6 +1987,15 @@ typedef enum { #define download_schedule_increment_bitfield_t \ ENUM_BF(download_schedule_increment_t) +/** Enumeration: do we want to use the random exponential backoff + * mechanism? */ +typedef enum { + DL_SCHED_DETERMINISTIC = 0, + DL_SCHED_RANDOM_EXPONENTIAL = 1, +} download_schedule_backoff_t; +#define download_schedule_backoff_bitfield_t \ + ENUM_BF(download_schedule_backoff_t) + /** Information about our plans for retrying downloads for a downloadable * directory object. * Each type of downloadable directory object has a corresponding retry @@ -2036,6 +2042,15 @@ typedef struct download_status_t { download_schedule_increment_bitfield_t increment_on : 1; /**< does this * schedule increment on each attempt, * or after each failure? */ + download_schedule_backoff_bitfield_t backoff : 1; /**< do we use the + * deterministic schedule, or random + * exponential backoffs? */ + uint8_t last_backoff_position; /**< number of attempts/failures, depending + * on increment_on, when we last recalculated + * the delay. Only updated if backoff + * == 1. */ + int last_delay_used; /**< last delay used for random exponential backoff; + * only updated if backoff == 1 */ } download_status_t; /** If n_download_failures is this high, the download can never happen. */ @@ -2508,6 +2523,18 @@ typedef struct networkstatus_voter_info_t { smartlist_t *sigs; } networkstatus_voter_info_t; +typedef struct networkstatus_sr_info_t { + /* Indicate if the dirauth partitipates in the SR protocol with its vote. + * This is tied to the SR flag in the vote. */ + unsigned int participate:1; + /* Both vote and consensus: Current and previous SRV. If list is empty, + * this means none were found in either the consensus or vote. */ + struct sr_srv_t *previous_srv; + struct sr_srv_t *current_srv; + /* Vote only: List of commitments. */ + smartlist_t *commits; +} networkstatus_sr_info_t; + /** Enumerates the possible seriousness values of a networkstatus document. */ typedef enum { NS_TYPE_VOTE, @@ -2590,6 +2617,9 @@ typedef struct networkstatus_t { /** If present, a map from descriptor digest to elements of * routerstatus_list. */ digestmap_t *desc_digest_map; + + /** Contains the shared random protocol data from a vote or consensus. */ + networkstatus_sr_info_t sr_info; } networkstatus_t; /** A set of signatures for a networkstatus consensus. Unless otherwise @@ -2958,17 +2988,17 @@ typedef struct circuit_t { /** When the circuit was first used, or 0 if the circuit is clean. * - * XXXX023 Note that some code will artifically adjust this value backward + * XXXX Note that some code will artifically adjust this value backward * in time in order to indicate that a circuit shouldn't be used for new * streams, but that it can stay alive as long as it has streams on it. * That's a kludge we should fix. * - * XXX023 The CBT code uses this field to record when HS-related + * XXX The CBT code uses this field to record when HS-related * circuits entered certain states. This usage probably won't * interfere with this field's primary purpose, but we should * document it more thoroughly to make sure of that. * - * XXX027 The SocksPort option KeepaliveIsolateSOCKSAuth will artificially + * XXX The SocksPort option KeepaliveIsolateSOCKSAuth will artificially * adjust this value forward each time a suitable stream is attached to an * already constructed circuit, potentially keeping the circuit alive * indefinitely. @@ -4483,6 +4513,17 @@ typedef struct { /** Autobool: Do we try to retain capabilities if we can? */ int KeepBindCapabilities; + + /** Maximum total size of unparseable descriptors to log during the + * lifetime of this Tor process. + */ + uint64_t MaxUnparseableDescSizeToLog; + + /** Bool (default: 1): Switch for the shared random protocol. Only + * relevant to a directory authority. If off, the authority won't + * participate in the protocol. If on (default), a flag is added to the + * vote indicating participation. */ + int AuthDirSharedRandomness; } or_options_t; /** Persistent state for an onion router, as saved to disk. */ @@ -5032,7 +5073,7 @@ typedef enum { /** Hidden-service side configuration of client authorization. */ typedef struct rend_authorized_client_t { char *client_name; - char descriptor_cookie[REND_DESC_COOKIE_LEN]; + uint8_t descriptor_cookie[REND_DESC_COOKIE_LEN]; crypto_pk_t *client_key; } rend_authorized_client_t; @@ -5060,12 +5101,12 @@ typedef struct rend_encoded_v2_service_descriptor_t { * INTRO_POINT_LIFETIME_INTRODUCTIONS INTRODUCE2 cells, it may expire * sooner.) * - * XXX023 Should this be configurable? */ + * XXX Should this be configurable? */ #define INTRO_POINT_LIFETIME_MIN_SECONDS (18*60*60) /** The maximum number of seconds that an introduction point will last * before expiring due to old age. * - * XXX023 Should this be configurable? */ + * XXX Should this be configurable? */ #define INTRO_POINT_LIFETIME_MAX_SECONDS (24*60*60) /** The maximum number of circuit creation retry we do to an intro point diff --git a/src/or/policies.c b/src/or/policies.c index f9718b6a95..7ddebd6096 100644 --- a/src/or/policies.c +++ b/src/or/policies.c @@ -103,7 +103,7 @@ policy_expand_private(smartlist_t **policy) if (tor_addr_parse_mask_ports(private_nets[i], 0, &newpolicy.addr, &newpolicy.maskbits, &port_min, &port_max)<0) { - tor_assert(0); + tor_assert_unreached(); } smartlist_add(tmp, addr_policy_get_canonical_entry(&newpolicy)); } @@ -2345,11 +2345,13 @@ policy_summary_reject(smartlist_t *summary, } /** Add a single exit policy item to our summary: - * If it is an accept ignore it unless it is for all IP addresses - * ("*"), i.e. it's prefixlen/maskbits is 0, else call + * + * If it is an accept, ignore it unless it is for all IP addresses + * ("*", i.e. its prefixlen/maskbits is 0). Otherwise call * policy_summary_accept(). - * If it's a reject ignore it if it is about one of the private - * networks, else call policy_summary_reject(). + * + * If it is a reject, ignore it if it is about one of the private + * networks. Otherwise call policy_summary_reject(). */ static void policy_summary_add_item(smartlist_t *summary, addr_policy_t *p) diff --git a/src/or/rendcache.c b/src/or/rendcache.c index f8206cd53b..e61a96b677 100644 --- a/src/or/rendcache.c +++ b/src/or/rendcache.c @@ -956,25 +956,25 @@ rend_cache_store_v2_desc_as_client(const char *desc, * avoid an evil HSDir serving old descriptor. We validate if the * timestamp is greater than and not equal because it's a rounded down * timestamp to the hour so if the descriptor changed in the same hour, - * the rend cache failure will tells us if we have a new descriptor. */ + * the rend cache failure will tell us if we have a new descriptor. */ if (e && e->parsed->timestamp > parsed->timestamp) { log_info(LD_REND, "We already have a new enough service descriptor for " "service ID %s with the same desc ID and version.", safe_str_client(service_id)); goto okay; } - /* Lookup our failure cache for intro point that might be unsuable. */ + /* Lookup our failure cache for intro point that might be unusable. */ validate_intro_point_failure(parsed, service_id); - /* It's now possible that our intro point list is empty, this means that + /* It's now possible that our intro point list is empty, which means that * this descriptor is useless to us because intro points have all failed * somehow before. Discard the descriptor. */ if (smartlist_len(parsed->intro_nodes) == 0) { - log_info(LD_REND, "Service descriptor with service ID %s, every " - "intro points are unusable. Discarding it.", + log_info(LD_REND, "Service descriptor with service ID %s has no " + "usable intro points. Discarding it.", safe_str_client(service_id)); goto err; } - /* Now either purge the current one and replace it's content or create a + /* Now either purge the current one and replace its content or create a * new one and add it to the rend cache. */ if (!e) { e = tor_malloc_zero(sizeof(rend_cache_entry_t)); diff --git a/src/or/rendcache.h b/src/or/rendcache.h index 0e8b918753..270b614c38 100644 --- a/src/or/rendcache.h +++ b/src/or/rendcache.h @@ -102,6 +102,13 @@ STATIC void validate_intro_point_failure(const rend_service_descriptor_t *desc, const char *service_id); STATIC void rend_cache_failure_entry_free_(void *entry); + +#ifdef TOR_UNIT_TESTS +extern strmap_t *rend_cache; +extern strmap_t *rend_cache_failure; +extern digestmap_t *rend_cache_v2_dir; +extern size_t rend_cache_total_allocation; +#endif #endif #endif /* TOR_RENDCACHE_H */ diff --git a/src/or/rendclient.c b/src/or/rendclient.c index 609c45c71d..64d367354b 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -510,7 +510,7 @@ lookup_last_hid_serv_request(routerstatus_t *hs_dir, tor_snprintf(hsdir_desc_comb_id, sizeof(hsdir_desc_comb_id), "%s%s", hsdir_id_base32, desc_id_base32); - /* XXX023 tor_assert(strlen(hsdir_desc_comb_id) == + /* XXX++?? tor_assert(strlen(hsdir_desc_comb_id) == LAST_HID_SERV_REQUEST_KEY_LEN); */ if (set) { time_t *oldptr; @@ -572,7 +572,7 @@ purge_hid_serv_from_last_hid_serv_requests(const char *desc_id) const char *key; void *val; strmap_iter_get(iter, &key, &val); - /* XXX023 tor_assert(strlen(key) == LAST_HID_SERV_REQUEST_KEY_LEN); */ + /* XXX++?? tor_assert(strlen(key) == LAST_HID_SERV_REQUEST_KEY_LEN); */ if (tor_memeq(key + LAST_HID_SERV_REQUEST_KEY_LEN - REND_DESC_ID_V2_LEN_BASE32, desc_id_base32, @@ -895,12 +895,6 @@ rend_client_refetch_v2_renddesc(rend_data_t *rend_query) rend_cache_entry_t *e = NULL; tor_assert(rend_query); - /* Are we configured to fetch descriptors? */ - if (!get_options()->FetchHidServDescriptors) { - log_warn(LD_REND, "We received an onion address for a v2 rendezvous " - "service descriptor, but are not fetching service descriptors."); - return; - } /* Before fetching, check if we already have a usable descriptor here. */ if (rend_cache_lookup_entry(rend_query->onion_address, -1, &e) == 0 && rend_client_any_intro_points_usable(e)) { @@ -908,6 +902,12 @@ rend_client_refetch_v2_renddesc(rend_data_t *rend_query) "already have a usable descriptor here. Not fetching."); return; } + /* Are we configured to fetch descriptors? */ + if (!get_options()->FetchHidServDescriptors) { + log_warn(LD_REND, "We received an onion address for a v2 rendezvous " + "service descriptor, but are not fetching service descriptors."); + return; + } log_debug(LD_REND, "Fetching v2 rendezvous descriptor for service %s", safe_str_client(rend_query->onion_address)); @@ -1099,7 +1099,7 @@ rend_client_rendezvous_acked(origin_circuit_t *circ, const uint8_t *request, * service and never reply to the client's rend requests */ pathbias_mark_use_success(circ); - /* XXXX This is a pretty brute-force approach. It'd be better to + /* XXXX++ This is a pretty brute-force approach. It'd be better to * attach only the connections that are waiting on this circuit, rather * than trying to attach them all. See comments bug 743. */ /* If we already have the introduction circuit built, make sure we send @@ -1466,12 +1466,10 @@ rend_parse_service_authorization(const or_options_t *options, strmap_t *parsed = strmap_new(); smartlist_t *sl = smartlist_new(); rend_service_authorization_t *auth = NULL; - char descriptor_cookie_tmp[REND_DESC_COOKIE_LEN+2]; - char descriptor_cookie_base64ext[REND_DESC_COOKIE_LEN_BASE64+2+1]; + char *err_msg = NULL; for (line = options->HidServAuth; line; line = line->next) { char *onion_address, *descriptor_cookie; - int auth_type_val = 0; auth = NULL; SMARTLIST_FOREACH(sl, char *, c, tor_free(c);); smartlist_clear(sl); @@ -1500,31 +1498,13 @@ rend_parse_service_authorization(const or_options_t *options, } /* Parse descriptor cookie. */ descriptor_cookie = smartlist_get(sl, 1); - if (strlen(descriptor_cookie) != REND_DESC_COOKIE_LEN_BASE64) { - log_warn(LD_CONFIG, "Authorization cookie has wrong length: '%s'", - descriptor_cookie); - goto err; - } - /* Add trailing zero bytes (AA) to make base64-decoding happy. */ - tor_snprintf(descriptor_cookie_base64ext, - REND_DESC_COOKIE_LEN_BASE64+2+1, - "%sAA", descriptor_cookie); - if (base64_decode(descriptor_cookie_tmp, sizeof(descriptor_cookie_tmp), - descriptor_cookie_base64ext, - strlen(descriptor_cookie_base64ext)) < 0) { - log_warn(LD_CONFIG, "Decoding authorization cookie failed: '%s'", - descriptor_cookie); - goto err; - } - auth_type_val = (((uint8_t)descriptor_cookie_tmp[16]) >> 4) + 1; - if (auth_type_val < 1 || auth_type_val > 2) { - log_warn(LD_CONFIG, "Authorization cookie has unknown authorization " - "type encoded."); + if (rend_auth_decode_cookie(descriptor_cookie, auth->descriptor_cookie, + &auth->auth_type, &err_msg) < 0) { + tor_assert(err_msg); + log_warn(LD_CONFIG, "%s", err_msg); + tor_free(err_msg); goto err; } - auth->auth_type = auth_type_val == 1 ? REND_BASIC_AUTH : REND_STEALTH_AUTH; - memcpy(auth->descriptor_cookie, descriptor_cookie_tmp, - REND_DESC_COOKIE_LEN); if (strmap_get(parsed, auth->onion_address)) { log_warn(LD_CONFIG, "Duplicate authorization for the same hidden " "service."); @@ -1547,8 +1527,6 @@ rend_parse_service_authorization(const or_options_t *options, } else { strmap_free(parsed, rend_service_authorization_strmap_item_free); } - memwipe(descriptor_cookie_tmp, 0, sizeof(descriptor_cookie_tmp)); - memwipe(descriptor_cookie_base64ext, 0, sizeof(descriptor_cookie_base64ext)); return res; } diff --git a/src/or/rendcommon.c b/src/or/rendcommon.c index 438fbc4d9a..01b0766cf0 100644 --- a/src/or/rendcommon.c +++ b/src/or/rendcommon.c @@ -211,7 +211,7 @@ rend_encode_v2_intro_points(char **encoded, rend_service_descriptor_t *desc) goto done; } /* Assemble everything for this introduction point. */ - address = tor_dup_addr(&info->addr); + address = tor_addr_to_str_dup(&info->addr); res = tor_snprintf(unenc + unenc_written, unenc_len - unenc_written, "introduction-point %s\n" "ip-address %s\n" @@ -720,6 +720,22 @@ rend_valid_descriptor_id(const char *query) return 0; } +/** Return true iff <b>client_name</b> is a syntactically valid name + * for rendezvous client authentication. */ +int +rend_valid_client_name(const char *client_name) +{ + size_t len = strlen(client_name); + if (len < 1 || len > REND_CLIENTNAME_MAX_LEN) { + return 0; + } + if (strspn(client_name, REND_LEGAL_CLIENTNAME_CHARACTERS) != len) { + return 0; + } + + return 1; +} + /** Called when we get a rendezvous-related relay cell on circuit * <b>circ</b>. Dispatch on rendezvous relay command. */ void @@ -941,3 +957,113 @@ hid_serv_get_responsible_directories(smartlist_t *responsible_dirs, return smartlist_len(responsible_dirs) ? 0 : -1; } +/* Length of the 'extended' auth cookie used to encode auth type before + * base64 encoding. */ +#define REND_DESC_COOKIE_LEN_EXT (REND_DESC_COOKIE_LEN + 1) +/* Length of the zero-padded auth cookie when base64 encoded. These two + * padding bytes always (A=) are stripped off of the returned cookie. */ +#define REND_DESC_COOKIE_LEN_EXT_BASE64 (REND_DESC_COOKIE_LEN_BASE64 + 2) + +/** Encode a client authorization descriptor cookie. + * The result of this function is suitable for use in the HidServAuth + * option. The trailing padding characters are removed, and the + * auth type is encoded into the cookie. + * + * Returns a new base64-encoded cookie. This function cannot fail. + * The caller is responsible for freeing the returned value. + */ +char * +rend_auth_encode_cookie(const uint8_t *cookie_in, rend_auth_type_t auth_type) +{ + uint8_t extended_cookie[REND_DESC_COOKIE_LEN_EXT]; + char *cookie_out = tor_malloc_zero(REND_DESC_COOKIE_LEN_EXT_BASE64 + 1); + int re; + + tor_assert(cookie_in); + + memcpy(extended_cookie, cookie_in, REND_DESC_COOKIE_LEN); + extended_cookie[REND_DESC_COOKIE_LEN] = ((int)auth_type - 1) << 4; + re = base64_encode(cookie_out, REND_DESC_COOKIE_LEN_EXT_BASE64 + 1, + (const char *) extended_cookie, REND_DESC_COOKIE_LEN_EXT, + 0); + tor_assert(re == REND_DESC_COOKIE_LEN_EXT_BASE64); + + /* Remove the trailing 'A='. Auth type is encoded in the high bits + * of the last byte, so the last base64 character will always be zero + * (A). This is subtly different behavior from base64_encode_nopad. */ + cookie_out[REND_DESC_COOKIE_LEN_BASE64] = '\0'; + memwipe(extended_cookie, 0, sizeof(extended_cookie)); + return cookie_out; +} + +/** Decode a base64-encoded client authorization descriptor cookie. + * The descriptor_cookie can be truncated to REND_DESC_COOKIE_LEN_BASE64 + * characters (as given to clients), or may include the two padding + * characters (as stored by the service). + * + * The result is stored in REND_DESC_COOKIE_LEN bytes of cookie_out. + * The rend_auth_type_t decoded from the cookie is stored in the + * optional auth_type_out parameter. + * + * Return 0 on success, or -1 on error. The caller is responsible for + * freeing the returned err_msg. + */ +int +rend_auth_decode_cookie(const char *cookie_in, uint8_t *cookie_out, + rend_auth_type_t *auth_type_out, char **err_msg_out) +{ + uint8_t descriptor_cookie_decoded[REND_DESC_COOKIE_LEN_EXT + 1] = { 0 }; + char descriptor_cookie_base64ext[REND_DESC_COOKIE_LEN_EXT_BASE64 + 1]; + const char *descriptor_cookie = cookie_in; + char *err_msg = NULL; + int auth_type_val = 0; + int res = -1; + int decoded_len; + + size_t len = strlen(descriptor_cookie); + if (len == REND_DESC_COOKIE_LEN_BASE64) { + /* Add a trailing zero byte to make base64-decoding happy. */ + tor_snprintf(descriptor_cookie_base64ext, + sizeof(descriptor_cookie_base64ext), + "%sA=", descriptor_cookie); + descriptor_cookie = descriptor_cookie_base64ext; + } else if (len != REND_DESC_COOKIE_LEN_EXT_BASE64) { + tor_asprintf(&err_msg, "Authorization cookie has wrong length: %s", + escaped(cookie_in)); + goto err; + } + + decoded_len = base64_decode((char *) descriptor_cookie_decoded, + sizeof(descriptor_cookie_decoded), + descriptor_cookie, + REND_DESC_COOKIE_LEN_EXT_BASE64); + if (decoded_len != REND_DESC_COOKIE_LEN && + decoded_len != REND_DESC_COOKIE_LEN_EXT) { + tor_asprintf(&err_msg, "Authorization cookie has invalid characters: %s", + escaped(cookie_in)); + goto err; + } + + if (auth_type_out) { + auth_type_val = (descriptor_cookie_decoded[REND_DESC_COOKIE_LEN] >> 4) + 1; + if (auth_type_val < 1 || auth_type_val > 2) { + tor_asprintf(&err_msg, "Authorization cookie type is unknown: %s", + escaped(cookie_in)); + goto err; + } + *auth_type_out = auth_type_val == 1 ? REND_BASIC_AUTH : REND_STEALTH_AUTH; + } + + memcpy(cookie_out, descriptor_cookie_decoded, REND_DESC_COOKIE_LEN); + res = 0; + err: + if (err_msg_out) { + *err_msg_out = err_msg; + } else { + tor_free(err_msg); + } + memwipe(descriptor_cookie_decoded, 0, sizeof(descriptor_cookie_decoded)); + memwipe(descriptor_cookie_base64ext, 0, sizeof(descriptor_cookie_base64ext)); + return res; +} + diff --git a/src/or/rendcommon.h b/src/or/rendcommon.h index d67552e405..88cf512f4a 100644 --- a/src/or/rendcommon.h +++ b/src/or/rendcommon.h @@ -45,6 +45,7 @@ void rend_intro_point_free(rend_intro_point_t *intro); int rend_valid_service_id(const char *query); int rend_valid_descriptor_id(const char *query); +int rend_valid_client_name(const char *client_name); int rend_encode_v2_descriptors(smartlist_t *descs_out, rend_service_descriptor_t *desc, time_t now, uint8_t period, rend_auth_type_t auth_type, @@ -68,5 +69,13 @@ rend_data_t *rend_data_service_create(const char *onion_address, const char *pk_digest, const uint8_t *cookie, rend_auth_type_t auth_type); + +char *rend_auth_encode_cookie(const uint8_t *cookie_in, + rend_auth_type_t auth_type); +int rend_auth_decode_cookie(const char *cookie_in, + uint8_t *cookie_out, + rend_auth_type_t *auth_type_out, + char **err_msg_out); + #endif diff --git a/src/or/rendmid.c b/src/or/rendmid.c index a33ad92966..ca0ad7b0d4 100644 --- a/src/or/rendmid.c +++ b/src/or/rendmid.c @@ -309,7 +309,7 @@ rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request, goto err; } - if (request_len != REND_COOKIE_LEN+DH_KEY_LEN+DIGEST_LEN) { + if (request_len < REND_COOKIE_LEN) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Rejecting RENDEZVOUS1 cell with bad length (%d) on circuit %u.", (int)request_len, (unsigned)circ->p_circ_id); diff --git a/src/or/rendservice.c b/src/or/rendservice.c index b81a01c568..c50de83f7e 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -183,14 +183,15 @@ num_rend_services(void) } /** Helper: free storage held by a single service authorized client entry. */ -static void +void rend_authorized_client_free(rend_authorized_client_t *client) { if (!client) return; if (client->client_key) crypto_pk_free(client->client_key); - memwipe(client->client_name, 0, strlen(client->client_name)); + if (client->client_name) + memwipe(client->client_name, 0, strlen(client->client_name)); tor_free(client->client_name); memwipe(client->descriptor_cookie, 0, sizeof(client->descriptor_cookie)); tor_free(client); @@ -671,27 +672,17 @@ rend_config_services(const or_options_t *options, int validate_only) SMARTLIST_FOREACH_BEGIN(clients, const char *, client_name) { rend_authorized_client_t *client; - size_t len = strlen(client_name); - if (len < 1 || len > REND_CLIENTNAME_MAX_LEN) { + if (!rend_valid_client_name(client_name)) { log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains an " - "illegal client name: '%s'. Length must be " - "between 1 and %d characters.", + "illegal client name: '%s'. Names must be " + "between 1 and %d characters and contain " + "only [A-Za-z0-9+_-].", client_name, REND_CLIENTNAME_MAX_LEN); SMARTLIST_FOREACH(clients, char *, cp, tor_free(cp)); smartlist_free(clients); rend_service_free(service); return -1; } - if (strspn(client_name, REND_LEGAL_CLIENTNAME_CHARACTERS) != len) { - log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains an " - "illegal client name: '%s'. Valid " - "characters are [A-Za-z0-9+_-].", - client_name); - SMARTLIST_FOREACH(clients, char *, cp, tor_free(cp)); - smartlist_free(clients); - rend_service_free(service); - return -1; - } client = tor_malloc_zero(sizeof(rend_authorized_client_t)); client->client_name = tor_strdup(client_name); smartlist_add(service->clients, client); @@ -827,14 +818,17 @@ rend_config_services(const or_options_t *options, int validate_only) return 0; } -/** Add the ephemeral service <b>pk</b>/<b>ports</b> if possible, with +/** Add the ephemeral service <b>pk</b>/<b>ports</b> if possible, using + * client authorization <b>auth_type</b> and an optional list of + * rend_authorized_client_t in <b>auth_clients</b>, with * <b>max_streams_per_circuit</b> streams allowed per rendezvous circuit, * and circuit closure on max streams being exceeded set by * <b>max_streams_close_circuit</b>. * - * Regardless of sucess/failure, callers should not touch pk/ports after - * calling this routine, and may assume that correct cleanup has been done - * on failure. + * Ownership of pk, ports, and auth_clients is passed to this routine. + * Regardless of success/failure, callers should not touch these values + * after calling this routine, and may assume that correct cleanup has + * been done on failure. * * Return an appropriate rend_service_add_ephemeral_status_t. */ @@ -843,6 +837,8 @@ rend_service_add_ephemeral(crypto_pk_t *pk, smartlist_t *ports, int max_streams_per_circuit, int max_streams_close_circuit, + rend_auth_type_t auth_type, + smartlist_t *auth_clients, char **service_id_out) { *service_id_out = NULL; @@ -852,7 +848,8 @@ rend_service_add_ephemeral(crypto_pk_t *pk, rend_service_t *s = tor_malloc_zero(sizeof(rend_service_t)); s->directory = NULL; /* This indicates the service is ephemeral. */ s->private_key = pk; - s->auth_type = REND_NO_AUTH; + s->auth_type = auth_type; + s->clients = auth_clients; s->ports = ports; s->intro_period_started = time(NULL); s->n_intro_points_wanted = NUM_INTRO_POINTS_DEFAULT; @@ -868,6 +865,12 @@ rend_service_add_ephemeral(crypto_pk_t *pk, rend_service_free(s); return RSAE_BADVIRTPORT; } + if (s->auth_type != REND_NO_AUTH && + (!s->clients || smartlist_len(s->clients) == 0)) { + log_warn(LD_CONFIG, "At least one authorized client must be specified."); + rend_service_free(s); + return RSAE_BADAUTH; + } /* Enforcing pk/id uniqueness should be done by rend_service_load_keys(), but * it's not, see #14828. @@ -923,7 +926,6 @@ rend_service_del_ephemeral(const char *service_id) */ SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) { if (!circ->marked_for_close && - circ->state == CIRCUIT_STATE_OPEN && (circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO || circ->purpose == CIRCUIT_PURPOSE_S_INTRO)) { origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ); @@ -1156,7 +1158,6 @@ rend_service_load_auth_keys(rend_service_t *s, const char *hfname) strmap_t *parsed_clients = strmap_new(); FILE *cfile, *hfile; open_file_t *open_cfile = NULL, *open_hfile = NULL; - char extended_desc_cookie[REND_DESC_COOKIE_LEN+1]; char desc_cook_out[3*REND_DESC_COOKIE_LEN_BASE64+1]; char service_id[16+1]; char buf[1500]; @@ -1208,10 +1209,12 @@ rend_service_load_auth_keys(rend_service_t *s, const char *hfname) memcpy(client->descriptor_cookie, parsed->descriptor_cookie, REND_DESC_COOKIE_LEN); } else { - crypto_rand(client->descriptor_cookie, REND_DESC_COOKIE_LEN); + crypto_rand((char *) client->descriptor_cookie, REND_DESC_COOKIE_LEN); } + /* For compatibility with older tor clients, this does not + * truncate the padding characters, unlike rend_auth_encode_cookie. */ if (base64_encode(desc_cook_out, 3*REND_DESC_COOKIE_LEN_BASE64+1, - client->descriptor_cookie, + (char *) client->descriptor_cookie, REND_DESC_COOKIE_LEN, 0) < 0) { log_warn(LD_BUG, "Could not base64-encode descriptor cookie."); goto err; @@ -1272,6 +1275,8 @@ rend_service_load_auth_keys(rend_service_t *s, const char *hfname) log_warn(LD_BUG, "Could not write client entry."); goto err; } + } else { + strlcpy(service_id, s->service_id, sizeof(service_id)); } if (fputs(buf, cfile) < 0) { @@ -1280,27 +1285,18 @@ rend_service_load_auth_keys(rend_service_t *s, const char *hfname) goto err; } - /* Add line to hostname file. */ - if (s->auth_type == REND_BASIC_AUTH) { - /* Remove == signs (newline has been removed above). */ - desc_cook_out[strlen(desc_cook_out)-2] = '\0'; - tor_snprintf(buf, sizeof(buf),"%s.onion %s # client: %s\n", - s->service_id, desc_cook_out, client->client_name); - } else { - memcpy(extended_desc_cookie, client->descriptor_cookie, - REND_DESC_COOKIE_LEN); - extended_desc_cookie[REND_DESC_COOKIE_LEN] = - ((int)s->auth_type - 1) << 4; - if (base64_encode(desc_cook_out, 3*REND_DESC_COOKIE_LEN_BASE64+1, - extended_desc_cookie, - REND_DESC_COOKIE_LEN+1, 0) < 0) { - log_warn(LD_BUG, "Could not base64-encode descriptor cookie."); - goto err; - } - desc_cook_out[strlen(desc_cook_out)-2] = '\0'; /* Remove A=. */ - tor_snprintf(buf, sizeof(buf),"%s.onion %s # client: %s\n", - service_id, desc_cook_out, client->client_name); + /* Add line to hostname file. This is not the same encoding as in + * client_keys. */ + char *encoded_cookie = rend_auth_encode_cookie(client->descriptor_cookie, + s->auth_type); + if (!encoded_cookie) { + log_warn(LD_BUG, "Could not base64-encode descriptor cookie."); + goto err; } + tor_snprintf(buf, sizeof(buf), "%s.onion %s # client: %s\n", + service_id, encoded_cookie, client->client_name); + memwipe(encoded_cookie, 0, strlen(encoded_cookie)); + tor_free(encoded_cookie); if (fputs(buf, hfile)<0) { log_warn(LD_FS, "Could not append host entry to file: %s", @@ -1332,7 +1328,6 @@ rend_service_load_auth_keys(rend_service_t *s, const char *hfname) memwipe(buf, 0, sizeof(buf)); memwipe(desc_cook_out, 0, sizeof(desc_cook_out)); memwipe(service_id, 0, sizeof(service_id)); - memwipe(extended_desc_cookie, 0, sizeof(extended_desc_cookie)); return r; } diff --git a/src/or/rendservice.h b/src/or/rendservice.h index 101b37e18d..4966cb0302 100644 --- a/src/or/rendservice.h +++ b/src/or/rendservice.h @@ -106,8 +106,11 @@ rend_service_port_config_t *rend_service_parse_port_config(const char *string, char **err_msg_out); void rend_service_port_config_free(rend_service_port_config_t *p); +void rend_authorized_client_free(rend_authorized_client_t *client); + /** Return value from rend_service_add_ephemeral. */ typedef enum { + RSAE_BADAUTH = -5, /**< Invalid auth_type/auth_clients */ RSAE_BADVIRTPORT = -4, /**< Invalid VIRTPORT/TARGET(s) */ RSAE_ADDREXISTS = -3, /**< Onion address collision */ RSAE_BADPRIVKEY = -2, /**< Invalid public key */ @@ -118,6 +121,8 @@ rend_service_add_ephemeral_status_t rend_service_add_ephemeral(crypto_pk_t *pk, smartlist_t *ports, int max_streams_per_circuit, int max_streams_close_circuit, + rend_auth_type_t auth_type, + smartlist_t *auth_clients, char **service_id_out); int rend_service_del_ephemeral(const char *service_id); diff --git a/src/or/rephist.c b/src/or/rephist.c index 04ed7aef0f..8992571c52 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -604,7 +604,7 @@ rep_hist_get_weighted_time_known(const char *id, time_t when) int rep_hist_have_measured_enough_stability(void) { - /* XXXX023 This doesn't do so well when we change our opinion + /* XXXX++ This doesn't do so well when we change our opinion * as to whether we're tracking router stability. */ return started_tracking_stability < time(NULL) - 4*60*60; } @@ -1074,7 +1074,8 @@ rep_hist_load_mtbf_data(time_t now) if (mtbf_idx > i) i = mtbf_idx; } - if (base16_decode(digest, DIGEST_LEN, hexbuf, HEX_DIGEST_LEN) < 0) { + if (base16_decode(digest, DIGEST_LEN, + hexbuf, HEX_DIGEST_LEN) != DIGEST_LEN) { log_warn(LD_HIST, "Couldn't hex string %s", escaped(hexbuf)); continue; } @@ -2738,7 +2739,7 @@ bidi_map_ent_hash(const bidi_map_entry_t *entry) } HT_PROTOTYPE(bidimap, bidi_map_entry_t, node, bidi_map_ent_hash, - bidi_map_ent_eq); + bidi_map_ent_eq) HT_GENERATE2(bidimap, bidi_map_entry_t, node, bidi_map_ent_hash, bidi_map_ent_eq, 0.6, tor_reallocarray_, tor_free_) @@ -2933,7 +2934,7 @@ static time_t start_of_hs_stats_interval; * information needed. */ typedef struct hs_stats_t { /** How many relay cells have we seen as rendezvous points? */ - int64_t rp_relay_cells_seen; + uint64_t rp_relay_cells_seen; /** Set of unique public key digests we've seen this stat period * (could also be implemented as sorted smartlist). */ @@ -3074,16 +3075,20 @@ rep_hist_format_hs_stats(time_t now) int64_t obfuscated_cells_seen; int64_t obfuscated_onions_seen; - obfuscated_cells_seen = round_int64_to_next_multiple_of( - hs_stats->rp_relay_cells_seen, - REND_CELLS_BIN_SIZE); - obfuscated_cells_seen = add_laplace_noise(obfuscated_cells_seen, + uint64_t rounded_cells_seen + = round_uint64_to_next_multiple_of(hs_stats->rp_relay_cells_seen, + REND_CELLS_BIN_SIZE); + rounded_cells_seen = MIN(rounded_cells_seen, INT64_MAX); + obfuscated_cells_seen = add_laplace_noise((int64_t)rounded_cells_seen, crypto_rand_double(), REND_CELLS_DELTA_F, REND_CELLS_EPSILON); - obfuscated_onions_seen = round_int64_to_next_multiple_of(digestmap_size( - hs_stats->onions_seen_this_period), - ONIONS_SEEN_BIN_SIZE); - obfuscated_onions_seen = add_laplace_noise(obfuscated_onions_seen, + + uint64_t rounded_onions_seen = + round_uint64_to_next_multiple_of((size_t)digestmap_size( + hs_stats->onions_seen_this_period), + ONIONS_SEEN_BIN_SIZE); + rounded_onions_seen = MIN(rounded_onions_seen, INT64_MAX); + obfuscated_onions_seen = add_laplace_noise((int64_t)rounded_onions_seen, crypto_rand_double(), ONIONS_SEEN_DELTA_F, ONIONS_SEEN_EPSILON); @@ -3217,7 +3222,7 @@ rep_hist_free_all(void) rep_hist_desc_stats_term(); total_descriptor_downloads = 0; - tor_assert(rephist_total_alloc == 0); - tor_assert(rephist_total_num == 0); + tor_assert_nonfatal(rephist_total_alloc == 0); + tor_assert_nonfatal_once(rephist_total_num == 0); } diff --git a/src/or/rephist.h b/src/or/rephist.h index 145da97d02..ff4810a56d 100644 --- a/src/or/rephist.h +++ b/src/or/rephist.h @@ -112,5 +112,12 @@ void rep_hist_note_negotiated_link_proto(unsigned link_proto, int started_here); void rep_hist_log_link_protocol_counts(void); +extern uint64_t rephist_total_alloc; +extern uint32_t rephist_total_num; +#ifdef TOR_UNIT_TESTS +extern int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1]; +extern int onion_handshakes_assigned[MAX_ONION_HANDSHAKE_TYPE+1]; +#endif + #endif diff --git a/src/or/router.c b/src/or/router.c index 01316c1bc2..a671591ad7 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -40,8 +40,6 @@ * and uploading server descriptors, retrying OR connections. **/ -extern long stats_n_seconds_working; - /************************************************************/ /***** @@ -1054,7 +1052,8 @@ init_keys(void) log_info(LD_DIR, "adding my own v3 cert"); if (trusted_dirs_load_certs_from_string( cert->cache_info.signed_descriptor_body, - TRUSTED_DIRS_CERTS_SRC_SELF, 0)<0) { + TRUSTED_DIRS_CERTS_SRC_SELF, 0, + NULL)<0) { log_warn(LD_DIR, "Unable to parse my own v3 cert! Failing."); return -1; } @@ -1537,7 +1536,7 @@ MOCK_IMPL(int, server_mode,(const or_options_t *options)) { if (options->ClientOnly) return 0; - /* XXXX024 I believe we can kill off ORListenAddress here.*/ + /* XXXX I believe we can kill off ORListenAddress here.*/ return (options->ORPort_set || options->ORListenAddress); } diff --git a/src/or/routerkeys.c b/src/or/routerkeys.c index fba3491f2b..12de8c5e7a 100644 --- a/src/or/routerkeys.c +++ b/src/or/routerkeys.c @@ -115,7 +115,7 @@ read_encrypted_secret_key(ed25519_secret_key_t *out, while (1) { ssize_t pwlen = - do_getpass("Enter pasphrase for master key:", pwbuf, sizeof(pwbuf), 0, + do_getpass("Enter passphrase for master key:", pwbuf, sizeof(pwbuf), 0, get_options()); if (pwlen < 0) { saved_errno = EINVAL; diff --git a/src/or/routerlist.c b/src/or/routerlist.c index 620c32d641..6ea9d8b0d1 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -67,7 +67,7 @@ typedef struct cert_list_t cert_list_t; /* static function prototypes */ static int compute_weighted_bandwidths(const smartlist_t *sl, bandwidth_weight_rule_t rule, - u64_dbl_t **bandwidths_out); + double **bandwidths_out); static const routerstatus_t *router_pick_trusteddirserver_impl( const smartlist_t *sourcelist, dirinfo_type_t auth, int flags, int *n_busy_out); @@ -159,6 +159,9 @@ download_status_cert_init(download_status_t *dlstatus) dlstatus->schedule = DL_SCHED_CONSENSUS; dlstatus->want_authority = DL_WANT_ANY_DIRSERVER; dlstatus->increment_on = DL_SCHED_INCREMENT_FAILURE; + dlstatus->backoff = DL_SCHED_RANDOM_EXPONENTIAL; + dlstatus->last_backoff_position = 0; + dlstatus->last_delay_used = 0; /* Use the new schedule to set next_attempt_at */ download_status_reset(dlstatus); @@ -250,6 +253,112 @@ get_cert_list(const char *id_digest) return cl; } +/** Return a list of authority ID digests with potentially enumerable lists + * of download_status_t objects; used by controller GETINFO queries. + */ + +MOCK_IMPL(smartlist_t *, +list_authority_ids_with_downloads, (void)) +{ + smartlist_t *ids = smartlist_new(); + digestmap_iter_t *i; + const char *digest; + char *tmp; + void *cl; + + if (trusted_dir_certs) { + for (i = digestmap_iter_init(trusted_dir_certs); + !(digestmap_iter_done(i)); + i = digestmap_iter_next(trusted_dir_certs, i)) { + /* + * We always have at least dl_status_by_id to query, so no need to + * probe deeper than the existence of a cert_list_t. + */ + digestmap_iter_get(i, &digest, &cl); + tmp = tor_malloc(DIGEST_LEN); + memcpy(tmp, digest, DIGEST_LEN); + smartlist_add(ids, tmp); + } + } + /* else definitely no downlaods going since nothing even has a cert list */ + + return ids; +} + +/** Given an authority ID digest, return a pointer to the default download + * status, or NULL if there is no such entry in trusted_dir_certs */ + +MOCK_IMPL(download_status_t *, +id_only_download_status_for_authority_id, (const char *digest)) +{ + download_status_t *dl = NULL; + cert_list_t *cl; + + if (trusted_dir_certs) { + cl = digestmap_get(trusted_dir_certs, digest); + if (cl) { + dl = &(cl->dl_status_by_id); + } + } + + return dl; +} + +/** Given an authority ID digest, return a smartlist of signing key digests + * for which download_status_t is potentially queryable, or NULL if no such + * authority ID digest is known. */ + +MOCK_IMPL(smartlist_t *, +list_sk_digests_for_authority_id, (const char *digest)) +{ + smartlist_t *sks = NULL; + cert_list_t *cl; + dsmap_iter_t *i; + const char *sk_digest; + char *tmp; + download_status_t *dl; + + if (trusted_dir_certs) { + cl = digestmap_get(trusted_dir_certs, digest); + if (cl) { + sks = smartlist_new(); + if (cl->dl_status_map) { + for (i = dsmap_iter_init(cl->dl_status_map); + !(dsmap_iter_done(i)); + i = dsmap_iter_next(cl->dl_status_map, i)) { + /* Pull the digest out and add it to the list */ + dsmap_iter_get(i, &sk_digest, &dl); + tmp = tor_malloc(DIGEST_LEN); + memcpy(tmp, sk_digest, DIGEST_LEN); + smartlist_add(sks, tmp); + } + } + } + } + + return sks; +} + +/** Given an authority ID digest and a signing key digest, return the + * download_status_t or NULL if none exists. */ + +MOCK_IMPL(download_status_t *, + download_status_for_authority_id_and_sk, + (const char *id_digest, const char *sk_digest)) +{ + download_status_t *dl = NULL; + cert_list_t *cl = NULL; + + if (trusted_dir_certs) { + cl = digestmap_get(trusted_dir_certs, id_digest); + if (cl && cl->dl_status_map) { + dl = dsmap_get(cl->dl_status_map, sk_digest); + } + } + + return dl; +} + /** Release all space held by a cert_list_t */ static void cert_list_free(cert_list_t *cl) @@ -287,7 +396,7 @@ trusted_dirs_reload_certs(void) return 0; r = trusted_dirs_load_certs_from_string( contents, - TRUSTED_DIRS_CERTS_SRC_FROM_STORE, 1); + TRUSTED_DIRS_CERTS_SRC_FROM_STORE, 1, NULL); tor_free(contents); return r; } @@ -317,16 +426,20 @@ already_have_cert(authority_cert_t *cert) * or TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST. If <b>flush</b> is true, we * need to flush any changed certificates to disk now. Return 0 on success, * -1 if any certs fail to parse. + * + * If source_dir is non-NULL, it's the identity digest for a directory that + * we've just successfully retrieved certificates from, so try it first to + * fetch any missing certificates. */ - int trusted_dirs_load_certs_from_string(const char *contents, int source, - int flush) + int flush, const char *source_dir) { dir_server_t *ds; const char *s, *eos; int failure_code = 0; int from_store = (source == TRUSTED_DIRS_CERTS_SRC_FROM_STORE); + int added_trusted_cert = 0; for (s = contents; *s; s = eos) { authority_cert_t *cert = authority_cert_parse_from_string(s, &eos); @@ -386,6 +499,7 @@ trusted_dirs_load_certs_from_string(const char *contents, int source, } if (ds) { + added_trusted_cert = 1; log_info(LD_DIR, "Adding %s certificate for directory authority %s with " "signing key %s", from_store ? "cached" : "downloaded", ds->nickname, hex_str(cert->signing_key_digest,DIGEST_LEN)); @@ -430,8 +544,15 @@ trusted_dirs_load_certs_from_string(const char *contents, int source, trusted_dirs_flush_certs_to_disk(); /* call this even if failure_code is <0, since some certs might have - * succeeded. */ - networkstatus_note_certs_arrived(); + * succeeded, but only pass source_dir if there were no failures, + * and at least one more authority certificate was added to the store. + * This avoids retrying a directory that's serving bad or entirely duplicate + * certificates. */ + if (failure_code == 0 && added_trusted_cert) { + networkstatus_note_certs_arrived(source_dir); + } else { + networkstatus_note_certs_arrived(NULL); + } return failure_code; } @@ -713,14 +834,81 @@ authority_cert_dl_looks_uncertain(const char *id_digest) return n_failures >= N_AUTH_CERT_DL_FAILURES_TO_BUG_USER; } +/* Fetch the authority certificates specified in resource. + * If we are a bridge client, and node is a configured bridge, fetch from node + * using dir_hint as the fingerprint. Otherwise, if rs is not NULL, fetch from + * rs. Otherwise, fetch from a random directory mirror. */ +static void +authority_certs_fetch_resource_impl(const char *resource, + const char *dir_hint, + const node_t *node, + const routerstatus_t *rs) +{ + const or_options_t *options = get_options(); + int get_via_tor = purpose_needs_anonymity(DIR_PURPOSE_FETCH_CERTIFICATE, 0); + + /* Make sure bridge clients never connect to anything but a bridge */ + if (options->UseBridges) { + if (node && !node_is_a_configured_bridge(node)) { + /* If we're using bridges, and node is not a bridge, use a 3-hop path. */ + get_via_tor = 1; + } else if (!node) { + /* If we're using bridges, and there's no node, use a 3-hop path. */ + get_via_tor = 1; + } + } + + const dir_indirection_t indirection = get_via_tor ? DIRIND_ANONYMOUS + : DIRIND_ONEHOP; + + /* If we've just downloaded a consensus from a bridge, re-use that + * bridge */ + if (options->UseBridges && node && !get_via_tor) { + /* clients always make OR connections to bridges */ + tor_addr_port_t or_ap; + /* we are willing to use a non-preferred address if we need to */ + fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0, + &or_ap); + directory_initiate_command(&or_ap.addr, or_ap.port, + NULL, 0, /*no dirport*/ + dir_hint, + DIR_PURPOSE_FETCH_CERTIFICATE, + 0, + indirection, + resource, NULL, 0, 0); + return; + } + + if (rs) { + /* If we've just downloaded a consensus from a directory, re-use that + * directory */ + directory_initiate_command_routerstatus(rs, + DIR_PURPOSE_FETCH_CERTIFICATE, + 0, indirection, resource, NULL, + 0, 0); + return; + } + + /* Otherwise, we want certs from a random fallback or directory + * mirror, because they will almost always succeed. */ + directory_get_from_dirserver(DIR_PURPOSE_FETCH_CERTIFICATE, 0, + resource, PDS_RETRY_IF_NO_SERVERS, + DL_WANT_ANY_DIRSERVER); +} + /** Try to download any v3 authority certificates that we may be missing. If * <b>status</b> is provided, try to get all the ones that were used to sign * <b>status</b>. Additionally, try to have a non-expired certificate for * every V3 authority in trusted_dir_servers. Don't fetch certificates we * already have. + * + * If dir_hint is non-NULL, it's the identity digest for a directory that + * we've just successfully retrieved a consensus or certificates from, so try + * it first to fetch any missing certificates. **/ void -authority_certs_fetch_missing(networkstatus_t *status, time_t now) +authority_certs_fetch_missing(networkstatus_t *status, time_t now, + const char *dir_hint) { /* * The pending_id digestmap tracks pending certificate downloads by @@ -739,12 +927,13 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) smartlist_t *missing_cert_digests, *missing_id_digests; char *resource = NULL; cert_list_t *cl; - const int cache = directory_caches_unknown_auth_certs(get_options()); + const or_options_t *options = get_options(); + const int cache = directory_caches_unknown_auth_certs(options); fp_pair_t *fp_tmp = NULL; char id_digest_str[2*DIGEST_LEN+1]; char sk_digest_str[2*DIGEST_LEN+1]; - if (should_delay_dir_fetches(get_options(), NULL)) + if (should_delay_dir_fetches(options, NULL)) return; pending_cert = fp_pair_map_new(); @@ -785,7 +974,7 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) } SMARTLIST_FOREACH_END(cert); if (!found && download_status_is_ready(&(cl->dl_status_by_id), now, - get_options()->TestingCertMaxDownloadTries) && + options->TestingCertMaxDownloadTries) && !digestmap_get(pending_id, ds->v3_identity_digest)) { log_info(LD_DIR, "No current certificate known for authority %s " @@ -847,7 +1036,7 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) } if (download_status_is_ready_by_sk_in_cl( cl, sig->signing_key_digest, - now, get_options()->TestingCertMaxDownloadTries) && + now, options->TestingCertMaxDownloadTries) && !fp_pair_map_get_by_digests(pending_cert, voter->identity_digest, sig->signing_key_digest)) { @@ -884,6 +1073,46 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) } SMARTLIST_FOREACH_END(voter); } + /* Bridge clients look up the node for the dir_hint */ + const node_t *node = NULL; + /* All clients, including bridge clients, look up the routerstatus for the + * dir_hint */ + const routerstatus_t *rs = NULL; + + /* If we still need certificates, try the directory that just successfully + * served us a consensus or certificates. + * As soon as the directory fails to provide additional certificates, we try + * another, randomly selected directory. This avoids continual retries. + * (We only ever have one outstanding request per certificate.) + */ + if (dir_hint) { + if (options->UseBridges) { + /* Bridge clients try the nodelist. If the dir_hint is from an authority, + * or something else fetched over tor, we won't find the node here, but + * we will find the rs. */ + node = node_get_by_id(dir_hint); + } + + /* All clients try the consensus routerstatus, then the fallback + * routerstatus */ + rs = router_get_consensus_status_by_id(dir_hint); + if (!rs) { + /* This will also find authorities */ + const dir_server_t *ds = router_get_fallback_dirserver_by_digest( + dir_hint); + if (ds) { + rs = &ds->fake_status; + } + } + + if (!node && !rs) { + log_warn(LD_BUG, "Directory %s delivered a consensus, but %s" + "no routerstatus could be found for it.", + options->UseBridges ? "no node and " : "", + hex_str(dir_hint, DIGEST_LEN)); + } + } + /* Do downloads by identity digest */ if (smartlist_len(missing_id_digests) > 0) { int need_plus = 0; @@ -913,11 +1142,9 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) if (smartlist_len(fps) > 1) { resource = smartlist_join_strings(fps, "", 0, NULL); - /* We want certs from mirrors, because they will almost always succeed. - */ - directory_get_from_dirserver(DIR_PURPOSE_FETCH_CERTIFICATE, 0, - resource, PDS_RETRY_IF_NO_SERVERS, - DL_WANT_ANY_DIRSERVER); + /* node and rs are directories that just gave us a consensus or + * certificates */ + authority_certs_fetch_resource_impl(resource, dir_hint, node, rs); tor_free(resource); } /* else we didn't add any: they were all pending */ @@ -960,11 +1187,9 @@ authority_certs_fetch_missing(networkstatus_t *status, time_t now) if (smartlist_len(fp_pairs) > 1) { resource = smartlist_join_strings(fp_pairs, "", 0, NULL); - /* We want certs from mirrors, because they will almost always succeed. - */ - directory_get_from_dirserver(DIR_PURPOSE_FETCH_CERTIFICATE, 0, - resource, PDS_RETRY_IF_NO_SERVERS, - DL_WANT_ANY_DIRSERVER); + /* node and rs are directories that just gave us a consensus or + * certificates */ + authority_certs_fetch_resource_impl(resource, dir_hint, node, rs); tor_free(resource); } /* else they were all pending */ @@ -1420,8 +1645,8 @@ router_digest_is_fallback_dir(const char *digest) * v3 identity key hashes to <b>digest</b>, or NULL if no such authority * is known. */ -dir_server_t * -trusteddirserver_get_by_v3_auth_digest(const char *digest) +MOCK_IMPL(dir_server_t *, +trusteddirserver_get_by_v3_auth_digest, (const char *digest)) { if (!trusted_dir_servers) return NULL; @@ -1815,20 +2040,23 @@ dirserver_choose_by_weight(const smartlist_t *servers, double authority_weight) { int n = smartlist_len(servers); int i; - u64_dbl_t *weights; + double *weights_dbl; + uint64_t *weights_u64; const dir_server_t *ds; - weights = tor_calloc(n, sizeof(u64_dbl_t)); + weights_dbl = tor_calloc(n, sizeof(double)); + weights_u64 = tor_calloc(n, sizeof(uint64_t)); for (i = 0; i < n; ++i) { ds = smartlist_get(servers, i); - weights[i].dbl = ds->weight; + weights_dbl[i] = ds->weight; if (ds->is_authority) - weights[i].dbl *= authority_weight; + weights_dbl[i] *= authority_weight; } - scale_array_elements_to_u64(weights, n, NULL); - i = choose_array_element_by_weight(weights, n); - tor_free(weights); + scale_array_elements_to_u64(weights_u64, weights_dbl, n, NULL); + i = choose_array_element_by_weight(weights_u64, n); + tor_free(weights_dbl); + tor_free(weights_u64); return (i < 0) ? NULL : smartlist_get(servers, i); } @@ -2090,59 +2318,43 @@ router_get_advertised_bandwidth_capped(const routerinfo_t *router) * much of the range of uint64_t. If <b>total_out</b> is provided, set it to * the sum of all elements in the array _before_ scaling. */ STATIC void -scale_array_elements_to_u64(u64_dbl_t *entries, int n_entries, +scale_array_elements_to_u64(uint64_t *entries_out, const double *entries_in, + int n_entries, uint64_t *total_out) { double total = 0.0; double scale_factor = 0.0; int i; - /* big, but far away from overflowing an int64_t */ -#define SCALE_TO_U64_MAX ((int64_t) (INT64_MAX / 4)) for (i = 0; i < n_entries; ++i) - total += entries[i].dbl; + total += entries_in[i]; - if (total > 0.0) - scale_factor = SCALE_TO_U64_MAX / total; + if (total > 0.0) { + scale_factor = ((double)INT64_MAX) / total; + scale_factor /= 4.0; /* make sure we're very far away from overflowing */ + } for (i = 0; i < n_entries; ++i) - entries[i].u64 = tor_llround(entries[i].dbl * scale_factor); + entries_out[i] = tor_llround(entries_in[i] * scale_factor); if (total_out) *total_out = (uint64_t) total; - -#undef SCALE_TO_U64_MAX } -/** Time-invariant 64-bit greater-than; works on two integers in the range - * (0,INT64_MAX). */ -#if SIZEOF_VOID_P == 8 -#define gt_i64_timei(a,b) ((a) > (b)) -#else -static inline int -gt_i64_timei(uint64_t a, uint64_t b) -{ - int64_t diff = (int64_t) (b - a); - int res = diff >> 63; - return res & 1; -} -#endif - /** Pick a random element of <b>n_entries</b>-element array <b>entries</b>, * choosing each element with a probability proportional to its (uint64_t) * value, and return the index of that element. If all elements are 0, choose * an index at random. Return -1 on error. */ STATIC int -choose_array_element_by_weight(const u64_dbl_t *entries, int n_entries) +choose_array_element_by_weight(const uint64_t *entries, int n_entries) { - int i, i_chosen=-1, n_chosen=0; - uint64_t total_so_far = 0; + int i; uint64_t rand_val; uint64_t total = 0; for (i = 0; i < n_entries; ++i) - total += entries[i].u64; + total += entries[i]; if (n_entries < 1) return -1; @@ -2154,22 +2366,8 @@ choose_array_element_by_weight(const u64_dbl_t *entries, int n_entries) rand_val = crypto_rand_uint64(total); - for (i = 0; i < n_entries; ++i) { - total_so_far += entries[i].u64; - if (gt_i64_timei(total_so_far, rand_val)) { - i_chosen = i; - n_chosen++; - /* Set rand_val to INT64_MAX rather than stopping the loop. This way, - * the time we spend in the loop does not leak which element we chose. */ - rand_val = INT64_MAX; - } - } - tor_assert(total_so_far == total); - tor_assert(n_chosen == 1); - tor_assert(i_chosen >= 0); - tor_assert(i_chosen < n_entries); - - return i_chosen; + return select_array_member_cumulative_timei( + entries, n_entries, total, rand_val); } /** When weighting bridges, enforce these values as lower and upper @@ -2221,17 +2419,21 @@ static const node_t * smartlist_choose_node_by_bandwidth_weights(const smartlist_t *sl, bandwidth_weight_rule_t rule) { - u64_dbl_t *bandwidths=NULL; + double *bandwidths_dbl=NULL; + uint64_t *bandwidths_u64=NULL; - if (compute_weighted_bandwidths(sl, rule, &bandwidths) < 0) + if (compute_weighted_bandwidths(sl, rule, &bandwidths_dbl) < 0) return NULL; - scale_array_elements_to_u64(bandwidths, smartlist_len(sl), NULL); + bandwidths_u64 = tor_calloc(smartlist_len(sl), sizeof(uint64_t)); + scale_array_elements_to_u64(bandwidths_u64, bandwidths_dbl, + smartlist_len(sl), NULL); { - int idx = choose_array_element_by_weight(bandwidths, + int idx = choose_array_element_by_weight(bandwidths_u64, smartlist_len(sl)); - tor_free(bandwidths); + tor_free(bandwidths_dbl); + tor_free(bandwidths_u64); return idx < 0 ? NULL : smartlist_get(sl, idx); } } @@ -2244,14 +2446,14 @@ smartlist_choose_node_by_bandwidth_weights(const smartlist_t *sl, static int compute_weighted_bandwidths(const smartlist_t *sl, bandwidth_weight_rule_t rule, - u64_dbl_t **bandwidths_out) + double **bandwidths_out) { int64_t weight_scale; double Wg = -1, Wm = -1, We = -1, Wd = -1; double Wgb = -1, Wmb = -1, Web = -1, Wdb = -1; uint64_t weighted_bw = 0; guardfraction_bandwidth_t guardfraction_bw; - u64_dbl_t *bandwidths; + double *bandwidths; /* Can't choose exit and guard at same time */ tor_assert(rule == NO_WEIGHTING || @@ -2333,7 +2535,7 @@ compute_weighted_bandwidths(const smartlist_t *sl, Web /= weight_scale; Wdb /= weight_scale; - bandwidths = tor_calloc(smartlist_len(sl), sizeof(u64_dbl_t)); + bandwidths = tor_calloc(smartlist_len(sl), sizeof(double)); // Cycle through smartlist and total the bandwidth. static int warned_missing_bw = 0; @@ -2420,7 +2622,7 @@ compute_weighted_bandwidths(const smartlist_t *sl, final_weight = weight*this_bw; } - bandwidths[node_sl_idx].dbl = final_weight + 0.5; + bandwidths[node_sl_idx] = final_weight + 0.5; } SMARTLIST_FOREACH_END(node); log_debug(LD_CIRC, "Generated weighted bandwidths for rule %s based " @@ -2441,7 +2643,7 @@ double frac_nodes_with_descriptors(const smartlist_t *sl, bandwidth_weight_rule_t rule) { - u64_dbl_t *bandwidths = NULL; + double *bandwidths = NULL; double total, present; if (smartlist_len(sl) == 0) @@ -2458,7 +2660,7 @@ frac_nodes_with_descriptors(const smartlist_t *sl, total = present = 0.0; SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) { - const double bw = bandwidths[node_sl_idx].dbl; + const double bw = bandwidths[node_sl_idx]; total += bw; if (node_has_descriptor(node)) present += bw; @@ -2632,7 +2834,8 @@ hex_digest_nickname_decode(const char *hexdigest, return -1; } - if (base16_decode(digest_out, DIGEST_LEN, hexdigest, HEX_DIGEST_LEN)<0) + if (base16_decode(digest_out, DIGEST_LEN, + hexdigest, HEX_DIGEST_LEN) != DIGEST_LEN) return -1; return 0; } @@ -2717,7 +2920,7 @@ hexdigest_to_digest(const char *hexdigest, char *digest) if (hexdigest[0]=='$') ++hexdigest; if (strlen(hexdigest) < HEX_DIGEST_LEN || - base16_decode(digest,DIGEST_LEN,hexdigest,HEX_DIGEST_LEN) < 0) + base16_decode(digest,DIGEST_LEN,hexdigest,HEX_DIGEST_LEN) != DIGEST_LEN) return -1; return 0; } @@ -3702,7 +3905,7 @@ router_add_extrainfo_to_routerlist(extrainfo_t *ei, const char **msg, was_router_added_t inserted; (void)from_fetch; if (msg) *msg = NULL; - /*XXXX023 Do something with msg */ + /*XXXX Do something with msg */ inserted = extrainfo_insert(router_get_routerlist(), ei, !from_cache); @@ -4305,7 +4508,7 @@ dir_server_new(int is_authority, return NULL; if (!hostname) - hostname_ = tor_dup_addr(addr); + hostname_ = tor_addr_to_str_dup(addr); else hostname_ = tor_strdup(hostname); @@ -4915,7 +5118,7 @@ update_consensus_router_descriptor_downloads(time_t now, int is_vote, /** How often should we launch a server/authority request to be sure of getting * a guess for our IP? */ -/*XXXX024 this info should come from netinfo cells or something, or we should +/*XXXX+ this info should come from netinfo cells or something, or we should * do this only when we aren't seeing incoming data. see bug 652. */ #define DUMMY_DOWNLOAD_INTERVAL (20*60) @@ -4926,7 +5129,7 @@ launch_dummy_descriptor_download_as_needed(time_t now, const or_options_t *options) { static time_t last_dummy_download = 0; - /* XXXX024 we could be smarter here; see notes on bug 652. */ + /* XXXX+ we could be smarter here; see notes on bug 652. */ /* If we're a server that doesn't have a configured address, we rely on * directory fetches to learn when our address changes. So if we haven't * tried to get any routerdescs in a long time, try a dummy fetch now. */ diff --git a/src/or/routerlist.h b/src/or/routerlist.h index 67cc253c5a..01f7644535 100644 --- a/src/or/routerlist.h +++ b/src/or/routerlist.h @@ -29,7 +29,7 @@ int trusted_dirs_reload_certs(void); #define TRUSTED_DIRS_CERTS_SRC_FROM_VOTE 4 int trusted_dirs_load_certs_from_string(const char *contents, int source, - int flush); + int flush, const char *source_dir); void trusted_dirs_flush_certs_to_disk(void); authority_cert_t *authority_cert_get_newest_by_id(const char *id_digest); authority_cert_t *authority_cert_get_by_sk_digest(const char *sk_digest); @@ -38,7 +38,8 @@ authority_cert_t *authority_cert_get_by_digests(const char *id_digest, void authority_cert_get_all(smartlist_t *certs_out); void authority_cert_dl_failed(const char *id_digest, const char *signing_key_digest, int status); -void authority_certs_fetch_missing(networkstatus_t *status, time_t now); +void authority_certs_fetch_missing(networkstatus_t *status, time_t now, + const char *dir_hint); int router_reload_router_list(void); int authority_cert_dl_looks_uncertain(const char *id_digest); const smartlist_t *router_get_trusted_dir_servers(void); @@ -51,7 +52,8 @@ dir_server_t *router_get_trusteddirserver_by_digest(const char *d); dir_server_t *router_get_fallback_dirserver_by_digest( const char *digest); int router_digest_is_fallback_dir(const char *digest); -dir_server_t *trusteddirserver_get_by_v3_auth_digest(const char *d); +MOCK_DECL(dir_server_t *, trusteddirserver_get_by_v3_auth_digest, + (const char *d)); const routerstatus_t *router_pick_trusteddirserver(dirinfo_type_t type, int flags); const routerstatus_t *router_pick_fallback_dirserver(dirinfo_type_t type, @@ -103,6 +105,14 @@ void routerlist_remove(routerlist_t *rl, routerinfo_t *ri, int make_old, void routerlist_free_all(void); void routerlist_reset_warnings(void); +MOCK_DECL(smartlist_t *, list_authority_ids_with_downloads, (void)); +MOCK_DECL(download_status_t *, id_only_download_status_for_authority_id, + (const char *digest)); +MOCK_DECL(smartlist_t *, list_sk_digests_for_authority_id, + (const char *digest)); +MOCK_DECL(download_status_t *, download_status_for_authority_id_and_sk, + (const char *id_digest, const char *sk_digest)); + static int WRA_WAS_ADDED(was_router_added_t s); static int WRA_WAS_OUTDATED(was_router_added_t s); static int WRA_WAS_REJECTED(was_router_added_t s); @@ -217,17 +227,11 @@ int hex_digest_nickname_matches(const char *hexdigest, const char *nickname, int is_named); #ifdef ROUTERLIST_PRIVATE -/** Helper type for choosing routers by bandwidth: contains a union of - * double and uint64_t. Before we call scale_array_elements_to_u64, it holds - * a double; after, it holds a uint64_t. */ -typedef union u64_dbl_t { - uint64_t u64; - double dbl; -} u64_dbl_t; - -STATIC int choose_array_element_by_weight(const u64_dbl_t *entries, +STATIC int choose_array_element_by_weight(const uint64_t *entries, int n_entries); -STATIC void scale_array_elements_to_u64(u64_dbl_t *entries, int n_entries, +STATIC void scale_array_elements_to_u64(uint64_t *entries_out, + const double *entries_in, + int n_entries, uint64_t *total_out); STATIC const routerstatus_t *router_pick_directory_server_impl( dirinfo_type_t auth, int flags, diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 91025c1568..0f9b9f75b6 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -28,6 +28,8 @@ #include "routerparse.h" #include "entrynodes.h" #include "torcert.h" +#include "sandbox.h" +#include "shared_random.h" #undef log #include <math.h> @@ -145,6 +147,11 @@ typedef enum { K_CONSENSUS_METHOD, K_LEGACY_DIR_KEY, K_DIRECTORY_FOOTER, + K_SIGNING_CERT_ED, + K_SR_FLAG, + K_COMMIT, + K_PREVIOUS_SRV, + K_CURRENT_SRV, K_PACKAGE, A_PURPOSE, @@ -446,6 +453,11 @@ static token_rule_t networkstatus_token_table[] = { T1("known-flags", K_KNOWN_FLAGS, ARGS, NO_OBJ ), T01("params", K_PARAMS, ARGS, NO_OBJ ), T( "fingerprint", K_FINGERPRINT, CONCAT_ARGS, NO_OBJ ), + T01("signing-ed25519", K_SIGNING_CERT_ED, NO_ARGS , NEED_OBJ ), + T01("shared-rand-participate",K_SR_FLAG, NO_ARGS, NO_OBJ ), + T0N("shared-rand-commit", K_COMMIT, GE(3), NO_OBJ ), + T01("shared-rand-previous-value", K_PREVIOUS_SRV,EQ(2), NO_OBJ ), + T01("shared-rand-current-value", K_CURRENT_SRV, EQ(2), NO_OBJ ), T0N("package", K_PACKAGE, CONCAT_ARGS, NO_OBJ ), CERTIFICATE_MEMBERS @@ -485,6 +497,9 @@ static token_rule_t networkstatus_consensus_token_table[] = { T01("consensus-method", K_CONSENSUS_METHOD, EQ(1), NO_OBJ), T01("params", K_PARAMS, ARGS, NO_OBJ ), + T01("shared-rand-previous-value", K_PREVIOUS_SRV, EQ(2), NO_OBJ ), + T01("shared-rand-current-value", K_CURRENT_SRV, EQ(2), NO_OBJ ), + END_OF_TABLE }; @@ -585,32 +600,579 @@ static int check_signature_token(const char *digest, #define DUMP_AREA(a,name) STMT_NIL #endif -/** Last time we dumped a descriptor to disk. */ -static time_t last_desc_dumped = 0; +/* Dump mechanism for unparseable descriptors */ + +/** List of dumped descriptors for FIFO cleanup purposes */ +STATIC smartlist_t *descs_dumped = NULL; +/** Total size of dumped descriptors for FIFO cleanup */ +STATIC uint64_t len_descs_dumped = 0; +/** Directory to stash dumps in */ +static int have_dump_desc_dir = 0; +static int problem_with_dump_desc_dir = 0; + +#define DESC_DUMP_DATADIR_SUBDIR "unparseable-descs" +#define DESC_DUMP_BASE_FILENAME "unparseable-desc" + +/** Find the dump directory and check if we'll be able to create it */ +static void +dump_desc_init(void) +{ + char *dump_desc_dir; + + dump_desc_dir = get_datadir_fname(DESC_DUMP_DATADIR_SUBDIR); + + /* + * We just check for it, don't create it at this point; we'll + * create it when we need it if it isn't already there. + */ + if (check_private_dir(dump_desc_dir, CPD_CHECK, get_options()->User) < 0) { + /* Error, log and flag it as having a problem */ + log_notice(LD_DIR, + "Doesn't look like we'll be able to create descriptor dump " + "directory %s; dumps will be disabled.", + dump_desc_dir); + problem_with_dump_desc_dir = 1; + tor_free(dump_desc_dir); + return; + } + + /* Check if it exists */ + switch (file_status(dump_desc_dir)) { + case FN_DIR: + /* We already have a directory */ + have_dump_desc_dir = 1; + break; + case FN_NOENT: + /* Nothing, we'll need to create it later */ + have_dump_desc_dir = 0; + break; + case FN_ERROR: + /* Log and flag having a problem */ + log_notice(LD_DIR, + "Couldn't check whether descriptor dump directory %s already" + " exists: %s", + dump_desc_dir, strerror(errno)); + problem_with_dump_desc_dir = 1; + break; + case FN_FILE: + case FN_EMPTY: + default: + /* Something else was here! */ + log_notice(LD_DIR, + "Descriptor dump directory %s already exists and isn't a " + "directory", + dump_desc_dir); + problem_with_dump_desc_dir = 1; + } + + if (have_dump_desc_dir && !problem_with_dump_desc_dir) { + dump_desc_populate_fifo_from_directory(dump_desc_dir); + } + + tor_free(dump_desc_dir); +} + +/** Create the dump directory if needed and possible */ +static void +dump_desc_create_dir(void) +{ + char *dump_desc_dir; + + /* If the problem flag is set, skip it */ + if (problem_with_dump_desc_dir) return; + + /* Do we need it? */ + if (!have_dump_desc_dir) { + dump_desc_dir = get_datadir_fname(DESC_DUMP_DATADIR_SUBDIR); + + if (check_private_dir(dump_desc_dir, CPD_CREATE, + get_options()->User) < 0) { + log_notice(LD_DIR, + "Failed to create descriptor dump directory %s", + dump_desc_dir); + problem_with_dump_desc_dir = 1; + } + + /* Okay, we created it */ + have_dump_desc_dir = 1; + + tor_free(dump_desc_dir); + } +} + +/** Dump desc FIFO/cleanup; take ownership of the given filename, add it to + * the FIFO, and clean up the oldest entries to the extent they exceed the + * configured cap. If any old entries with a matching hash existed, they + * just got overwritten right before this was called and we should adjust + * the total size counter without deleting them. + */ +static void +dump_desc_fifo_add_and_clean(char *filename, const uint8_t *digest_sha256, + size_t len) +{ + dumped_desc_t *ent = NULL, *tmp; + uint64_t max_len; + + tor_assert(filename != NULL); + tor_assert(digest_sha256 != NULL); + + if (descs_dumped == NULL) { + /* We better have no length, then */ + tor_assert(len_descs_dumped == 0); + /* Make a smartlist */ + descs_dumped = smartlist_new(); + } + + /* Make a new entry to put this one in */ + ent = tor_malloc_zero(sizeof(*ent)); + ent->filename = filename; + ent->len = len; + ent->when = time(NULL); + memcpy(ent->digest_sha256, digest_sha256, DIGEST256_LEN); + + /* Do we need to do some cleanup? */ + max_len = get_options()->MaxUnparseableDescSizeToLog; + /* Iterate over the list until we've freed enough space */ + while (len > max_len - len_descs_dumped && + smartlist_len(descs_dumped) > 0) { + /* Get the oldest thing on the list */ + tmp = (dumped_desc_t *)(smartlist_get(descs_dumped, 0)); + + /* + * Check if it matches the filename we just added, so we don't delete + * something we just emitted if we get repeated identical descriptors. + */ + if (strcmp(tmp->filename, filename) != 0) { + /* Delete it and adjust the length counter */ + tor_unlink(tmp->filename); + tor_assert(len_descs_dumped >= tmp->len); + len_descs_dumped -= tmp->len; + log_info(LD_DIR, + "Deleting old unparseable descriptor dump %s due to " + "space limits", + tmp->filename); + } else { + /* + * Don't delete, but do adjust the counter since we will bump it + * later + */ + tor_assert(len_descs_dumped >= tmp->len); + len_descs_dumped -= tmp->len; + log_info(LD_DIR, + "Replacing old descriptor dump %s with new identical one", + tmp->filename); + } + + /* Free it and remove it from the list */ + smartlist_del_keeporder(descs_dumped, 0); + tor_free(tmp->filename); + tor_free(tmp); + } + + /* Append our entry to the end of the list and bump the counter */ + smartlist_add(descs_dumped, ent); + len_descs_dumped += len; +} + +/** Check if we already have a descriptor for this hash and move it to the + * head of the queue if so. Return 1 if one existed and 0 otherwise. + */ +static int +dump_desc_fifo_bump_hash(const uint8_t *digest_sha256) +{ + dumped_desc_t *match = NULL; + + tor_assert(digest_sha256); + + if (descs_dumped) { + /* Find a match if one exists */ + SMARTLIST_FOREACH_BEGIN(descs_dumped, dumped_desc_t *, ent) { + if (ent && + tor_memeq(ent->digest_sha256, digest_sha256, DIGEST256_LEN)) { + /* + * Save a pointer to the match and remove it from its current + * position. + */ + match = ent; + SMARTLIST_DEL_CURRENT_KEEPORDER(descs_dumped, ent); + break; + } + } SMARTLIST_FOREACH_END(ent); + + if (match) { + /* Update the timestamp */ + match->when = time(NULL); + /* Add it back at the end of the list */ + smartlist_add(descs_dumped, match); + + /* Indicate we found one */ + return 1; + } + } + + return 0; +} + +/** Clean up on exit; just memory, leave the dumps behind + */ +STATIC void +dump_desc_fifo_cleanup(void) +{ + if (descs_dumped) { + /* Free each descriptor */ + SMARTLIST_FOREACH_BEGIN(descs_dumped, dumped_desc_t *, ent) { + tor_assert(ent); + tor_free(ent->filename); + tor_free(ent); + } SMARTLIST_FOREACH_END(ent); + /* Free the list */ + smartlist_free(descs_dumped); + descs_dumped = NULL; + len_descs_dumped = 0; + } +} + +/** Handle one file for dump_desc_populate_fifo_from_directory(); make sure + * the filename is sensibly formed and matches the file content, and either + * return a dumped_desc_t for it or remove the file and return NULL. + */ +MOCK_IMPL(STATIC dumped_desc_t *, +dump_desc_populate_one_file, (const char *dirname, const char *f)) +{ + dumped_desc_t *ent = NULL; + char *path = NULL, *desc = NULL; + const char *digest_str; + char digest[DIGEST256_LEN], content_digest[DIGEST256_LEN]; + /* Expected prefix before digest in filenames */ + const char *f_pfx = DESC_DUMP_BASE_FILENAME "."; + /* + * Stat while reading; this is important in case the file + * contains a NUL character. + */ + struct stat st; + + /* Sanity-check args */ + tor_assert(dirname != NULL); + tor_assert(f != NULL); + + /* Form the full path */ + tor_asprintf(&path, "%s" PATH_SEPARATOR "%s", dirname, f); + + /* Check that f has the form DESC_DUMP_BASE_FILENAME.<digest256> */ + + if (!strcmpstart(f, f_pfx)) { + /* It matches the form, but is the digest parseable as such? */ + digest_str = f + strlen(f_pfx); + if (base16_decode(digest, DIGEST256_LEN, + digest_str, strlen(digest_str)) != DIGEST256_LEN) { + /* We failed to decode it */ + digest_str = NULL; + } + } else { + /* No match */ + digest_str = NULL; + } + + if (!digest_str) { + /* We couldn't get a sensible digest */ + log_notice(LD_DIR, + "Removing unrecognized filename %s from unparseable " + "descriptors directory", f); + tor_unlink(path); + /* We're done */ + goto done; + } + + /* + * The filename has the form DESC_DUMP_BASE_FILENAME "." <digest256> and + * we've decoded the digest. Next, check that we can read it and the + * content matches this digest. We are relying on the fact that if the + * file contains a '\0', read_file_to_str() will allocate space for and + * read the entire file and return the correct size in st. + */ + desc = read_file_to_str(path, RFTS_IGNORE_MISSING|RFTS_BIN, &st); + if (!desc) { + /* We couldn't read it */ + log_notice(LD_DIR, + "Failed to read %s from unparseable descriptors directory; " + "attempting to remove it.", f); + tor_unlink(path); + /* We're done */ + goto done; + } + +#if SIZE_MAX > UINT64_MAX + if (BUG((uint64_t)st.st_size > (uint64_t)SIZE_MAX)) { + /* LCOV_EXCL_START + * Should be impossible since RFTS above should have failed to read the + * huge file into RAM. */ + goto done; + /* LCOV_EXCL_STOP */ + } +#endif + if (BUG(st.st_size < 0)) { + /* LCOV_EXCL_START + * Should be impossible, since the OS isn't supposed to be b0rken. */ + goto done; + /* LCOV_EXCL_STOP */ + } + /* (Now we can be sure that st.st_size is safe to cast to a size_t.) */ + + /* + * We got one; now compute its digest and check that it matches the + * filename. + */ + if (crypto_digest256((char *)content_digest, desc, (size_t) st.st_size, + DIGEST_SHA256) != 0) { + /* Weird, but okay */ + log_info(LD_DIR, + "Unable to hash content of %s from unparseable descriptors " + "directory", f); + tor_unlink(path); + /* We're done */ + goto done; + } + + /* Compare the digests */ + if (tor_memneq(digest, content_digest, DIGEST256_LEN)) { + /* No match */ + log_info(LD_DIR, + "Hash of %s from unparseable descriptors directory didn't " + "match its filename; removing it", f); + tor_unlink(path); + /* We're done */ + goto done; + } + + /* Okay, it's a match, we should prepare ent */ + ent = tor_malloc_zero(sizeof(dumped_desc_t)); + ent->filename = path; + memcpy(ent->digest_sha256, digest, DIGEST256_LEN); + ent->len = (size_t) st.st_size; + ent->when = st.st_mtime; + /* Null out path so we don't free it out from under ent */ + path = NULL; + + done: + /* Free allocations if we had them */ + tor_free(desc); + tor_free(path); + + return ent; +} + +/** Sort helper for dump_desc_populate_fifo_from_directory(); compares + * the when field of dumped_desc_ts in a smartlist to put the FIFO in + * the correct order after reconstructing it from the directory. + */ +static int +dump_desc_compare_fifo_entries(const void **a_v, const void **b_v) +{ + const dumped_desc_t **a = (const dumped_desc_t **)a_v; + const dumped_desc_t **b = (const dumped_desc_t **)b_v; + + if ((a != NULL) && (*a != NULL)) { + if ((b != NULL) && (*b != NULL)) { + /* We have sensible dumped_desc_ts to compare */ + if ((*a)->when < (*b)->when) { + return -1; + } else if ((*a)->when == (*b)->when) { + return 0; + } else { + return 1; + } + } else { + /* + * We shouldn't see this, but what the hell, NULLs precede everythin + * else + */ + return 1; + } + } else { + return -1; + } +} + +/** Scan the contents of the directory, and update FIFO/counters; this will + * consistency-check descriptor dump filenames against hashes of descriptor + * dump file content, and remove any inconsistent/unreadable dumps, and then + * reconstruct the dump FIFO as closely as possible for the last time the + * tor process shut down. If a previous dump was repeated more than once and + * moved ahead in the FIFO, the mtime will not have been updated and the + * reconstructed order will be wrong, but will always be a permutation of + * the original. + */ +STATIC void +dump_desc_populate_fifo_from_directory(const char *dirname) +{ + smartlist_t *files = NULL; + dumped_desc_t *ent = NULL; + + tor_assert(dirname != NULL); + + /* Get a list of files */ + files = tor_listdir(dirname); + if (!files) { + log_notice(LD_DIR, + "Unable to get contents of unparseable descriptor dump " + "directory %s", + dirname); + return; + } + + /* + * Iterate through the list and decide which files should go in the + * FIFO and which should be purged. + */ + + SMARTLIST_FOREACH_BEGIN(files, char *, f) { + /* Try to get a FIFO entry */ + ent = dump_desc_populate_one_file(dirname, f); + if (ent) { + /* + * We got one; add it to the FIFO. No need for duplicate checking + * here since we just verified the name and digest match. + */ + + /* Make sure we have a list to add it to */ + if (!descs_dumped) { + descs_dumped = smartlist_new(); + len_descs_dumped = 0; + } + + /* Add it and adjust the counter */ + smartlist_add(descs_dumped, ent); + len_descs_dumped += ent->len; + } + /* + * If we didn't, we will have unlinked the file if necessary and + * possible, and emitted a log message about it, so just go on to + * the next. + */ + } SMARTLIST_FOREACH_END(f); + + /* Did we get anything? */ + if (descs_dumped != NULL) { + /* Sort the FIFO in order of increasing timestamp */ + smartlist_sort(descs_dumped, dump_desc_compare_fifo_entries); + + /* Log some stats */ + log_info(LD_DIR, + "Reloaded unparseable descriptor dump FIFO with %d dump(s) " + "totaling " U64_FORMAT " bytes", + smartlist_len(descs_dumped), U64_PRINTF_ARG(len_descs_dumped)); + } + + /* Free the original list */ + SMARTLIST_FOREACH(files, char *, f, tor_free(f)); + smartlist_free(files); +} /** For debugging purposes, dump unparseable descriptor *<b>desc</b> of * type *<b>type</b> to file $DATADIR/unparseable-desc. Do not write more * than one descriptor to disk per minute. If there is already such a * file in the data directory, overwrite it. */ -static void +STATIC void dump_desc(const char *desc, const char *type) { - time_t now = time(NULL); tor_assert(desc); tor_assert(type); - if (!last_desc_dumped || last_desc_dumped + 60 < now) { - char *debugfile = get_datadir_fname("unparseable-desc"); - size_t filelen = 50 + strlen(type) + strlen(desc); - char *content = tor_malloc_zero(filelen); - tor_snprintf(content, filelen, "Unable to parse descriptor of type " - "%s:\n%s", type, desc); - write_str_to_file(debugfile, content, 1); - log_info(LD_DIR, "Unable to parse descriptor of type %s. See file " - "unparseable-desc in data directory for details.", type); - tor_free(content); - tor_free(debugfile); - last_desc_dumped = now; + size_t len; + /* The SHA256 of the string */ + uint8_t digest_sha256[DIGEST256_LEN]; + char digest_sha256_hex[HEX_DIGEST256_LEN+1]; + /* Filename to log it to */ + char *debugfile, *debugfile_base; + + /* Get the hash for logging purposes anyway */ + len = strlen(desc); + if (crypto_digest256((char *)digest_sha256, desc, len, + DIGEST_SHA256) != 0) { + log_info(LD_DIR, + "Unable to parse descriptor of type %s, and unable to even hash" + " it!", type); + goto err; + } + + base16_encode(digest_sha256_hex, sizeof(digest_sha256_hex), + (const char *)digest_sha256, sizeof(digest_sha256)); + + /* + * We mention type and hash in the main log; don't clutter up the files + * with anything but the exact dump. + */ + tor_asprintf(&debugfile_base, + DESC_DUMP_BASE_FILENAME ".%s", digest_sha256_hex); + debugfile = get_datadir_fname2(DESC_DUMP_DATADIR_SUBDIR, debugfile_base); + + /* + * Check if the sandbox is active or will become active; see comment + * below at the log message for why. + */ + if (!(sandbox_is_active() || get_options()->Sandbox)) { + if (len <= get_options()->MaxUnparseableDescSizeToLog) { + if (!dump_desc_fifo_bump_hash(digest_sha256)) { + /* Create the directory if needed */ + dump_desc_create_dir(); + /* Make sure we've got it */ + if (have_dump_desc_dir && !problem_with_dump_desc_dir) { + /* Write it, and tell the main log about it */ + write_str_to_file(debugfile, desc, 1); + log_info(LD_DIR, + "Unable to parse descriptor of type %s with hash %s and " + "length %lu. See file %s in data directory for details.", + type, digest_sha256_hex, (unsigned long)len, + debugfile_base); + dump_desc_fifo_add_and_clean(debugfile, digest_sha256, len); + /* Since we handed ownership over, don't free debugfile later */ + debugfile = NULL; + } else { + /* Problem with the subdirectory */ + log_info(LD_DIR, + "Unable to parse descriptor of type %s with hash %s and " + "length %lu. Descriptor not dumped because we had a " + "problem creating the " DESC_DUMP_DATADIR_SUBDIR + " subdirectory", + type, digest_sha256_hex, (unsigned long)len); + /* We do have to free debugfile in this case */ + } + } else { + /* We already had one with this hash dumped */ + log_info(LD_DIR, + "Unable to parse descriptor of type %s with hash %s and " + "length %lu. Descriptor not dumped because one with that " + "hash has already been dumped.", + type, digest_sha256_hex, (unsigned long)len); + /* We do have to free debugfile in this case */ + } + } else { + /* Just log that it happened without dumping */ + log_info(LD_DIR, + "Unable to parse descriptor of type %s with hash %s and " + "length %lu. Descriptor not dumped because it exceeds maximum" + " log size all by itself.", + type, digest_sha256_hex, (unsigned long)len); + /* We do have to free debugfile in this case */ + } + } else { + /* + * Not logging because the sandbox is active and seccomp2 apparently + * doesn't have a sensible way to allow filenames according to a pattern + * match. (If we ever figure out how to say "allow writes to /regex/", + * remove this checK). + */ + log_info(LD_DIR, + "Unable to parse descriptor of type %s with hash %s and " + "length %lu. Descriptor not dumped because the sandbox is " + "configured", + type, digest_sha256_hex, (unsigned long)len); } + + tor_free(debugfile_base); + tor_free(debugfile); + + err: + return; } /** Set <b>digest</b> to the SHA-1 digest of the hash of the directory in @@ -1513,7 +2075,8 @@ router_parse_entry_from_string(const char *s, const char *end, char d[DIGEST_LEN]; tor_assert(tok->n_args == 1); tor_strstrip(tok->args[0], " "); - if (base16_decode(d, DIGEST_LEN, tok->args[0], strlen(tok->args[0]))) { + if (base16_decode(d, DIGEST_LEN, + tok->args[0], strlen(tok->args[0])) != DIGEST_LEN) { log_warn(LD_DIR, "Couldn't decode router fingerprint %s", escaped(tok->args[0])); goto err; @@ -1594,8 +2157,10 @@ router_parse_entry_from_string(const char *s, const char *end, if ((tok = find_opt_by_keyword(tokens, K_EXTRA_INFO_DIGEST))) { tor_assert(tok->n_args >= 1); if (strlen(tok->args[0]) == HEX_DIGEST_LEN) { - base16_decode(router->cache_info.extra_info_digest, - DIGEST_LEN, tok->args[0], HEX_DIGEST_LEN); + if (base16_decode(router->cache_info.extra_info_digest, DIGEST_LEN, + tok->args[0], HEX_DIGEST_LEN) != DIGEST_LEN) { + log_warn(LD_DIR,"Invalid extra info digest"); + } } else { log_warn(LD_DIR, "Invalid extra info digest %s", escaped(tok->args[0])); } @@ -1738,7 +2303,7 @@ extrainfo_parse_entry_from_string(const char *s, const char *end, strlcpy(extrainfo->nickname, tok->args[0], sizeof(extrainfo->nickname)); if (strlen(tok->args[1]) != HEX_DIGEST_LEN || base16_decode(extrainfo->cache_info.identity_digest, DIGEST_LEN, - tok->args[1], HEX_DIGEST_LEN)) { + tok->args[1], HEX_DIGEST_LEN) != DIGEST_LEN) { log_warn(LD_DIR,"Invalid fingerprint %s on \"extra-info\"", escaped(tok->args[1])); goto err; @@ -1960,7 +2525,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) tok = find_by_keyword(tokens, K_FINGERPRINT); tor_assert(tok->n_args); if (base16_decode(fp_declared, DIGEST_LEN, tok->args[0], - strlen(tok->args[0]))) { + strlen(tok->args[0])) != DIGEST_LEN) { log_warn(LD_DIR, "Couldn't decode key certificate fingerprint %s", escaped(tok->args[0])); goto err; @@ -1981,7 +2546,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string) struct in_addr in; char *address = NULL; tor_assert(tok->n_args); - /* XXX024 use some tor_addr parse function below instead. -RD */ + /* XXX++ use some tor_addr parse function below instead. -RD */ if (tor_addr_port_split(LOG_WARN, tok->args[0], &address, &cert->dir_port) < 0 || tor_inet_aton(address, &in) == 0) { @@ -2840,6 +3405,134 @@ networkstatus_verify_bw_weights(networkstatus_t *ns, int consensus_method) return valid; } +/** Parse and extract all SR commits from <b>tokens</b> and place them in + * <b>ns</b>. */ +static void +extract_shared_random_commits(networkstatus_t *ns, smartlist_t *tokens) +{ + smartlist_t *chunks = NULL; + + tor_assert(ns); + tor_assert(tokens); + /* Commits are only present in a vote. */ + tor_assert(ns->type == NS_TYPE_VOTE); + + ns->sr_info.commits = smartlist_new(); + + smartlist_t *commits = find_all_by_keyword(tokens, K_COMMIT); + /* It's normal that a vote might contain no commits even if it participates + * in the SR protocol. Don't treat it as an error. */ + if (commits == NULL) { + goto end; + } + + /* Parse the commit. We do NO validation of number of arguments or ordering + * for forward compatibility, it's the parse commit job to inform us if it's + * supported or not. */ + chunks = smartlist_new(); + SMARTLIST_FOREACH_BEGIN(commits, directory_token_t *, tok) { + /* Extract all arguments and put them in the chunks list. */ + for (int i = 0; i < tok->n_args; i++) { + smartlist_add(chunks, tok->args[i]); + } + sr_commit_t *commit = sr_parse_commit(chunks); + smartlist_clear(chunks); + if (commit == NULL) { + /* Get voter identity so we can warn that this dirauth vote contains + * commit we can't parse. */ + networkstatus_voter_info_t *voter = smartlist_get(ns->voters, 0); + tor_assert(voter); + log_warn(LD_DIR, "SR: Unable to parse commit %s from vote of voter %s.", + escaped(tok->object_body), + hex_str(voter->identity_digest, + sizeof(voter->identity_digest))); + /* Commitment couldn't be parsed. Continue onto the next commit because + * this one could be unsupported for instance. */ + continue; + } + /* Add newly created commit object to the vote. */ + smartlist_add(ns->sr_info.commits, commit); + } SMARTLIST_FOREACH_END(tok); + + end: + smartlist_free(chunks); + smartlist_free(commits); +} + +/** Check if a shared random value of type <b>srv_type</b> is in + * <b>tokens</b>. If there is, parse it and set it to <b>srv_out</b>. Return + * -1 on failure, 0 on success. The resulting srv is allocated on the heap and + * it's the responsibility of the caller to free it. */ +static int +extract_one_srv(smartlist_t *tokens, directory_keyword srv_type, + sr_srv_t **srv_out) +{ + int ret = -1; + directory_token_t *tok; + sr_srv_t *srv = NULL; + smartlist_t *chunks; + + tor_assert(tokens); + + chunks = smartlist_new(); + tok = find_opt_by_keyword(tokens, srv_type); + if (!tok) { + /* That's fine, no SRV is allowed. */ + ret = 0; + goto end; + } + for (int i = 0; i < tok->n_args; i++) { + smartlist_add(chunks, tok->args[i]); + } + srv = sr_parse_srv(chunks); + if (srv == NULL) { + log_warn(LD_DIR, "SR: Unparseable SRV %s", escaped(tok->object_body)); + goto end; + } + /* All is good. */ + *srv_out = srv; + ret = 0; + end: + smartlist_free(chunks); + return ret; +} + +/** Extract any shared random values found in <b>tokens</b> and place them in + * the networkstatus <b>ns</b>. */ +static void +extract_shared_random_srvs(networkstatus_t *ns, smartlist_t *tokens) +{ + const char *voter_identity; + networkstatus_voter_info_t *voter; + + tor_assert(ns); + tor_assert(tokens); + /* Can be only one of them else code flow. */ + tor_assert(ns->type == NS_TYPE_VOTE || ns->type == NS_TYPE_CONSENSUS); + + if (ns->type == NS_TYPE_VOTE) { + voter = smartlist_get(ns->voters, 0); + tor_assert(voter); + voter_identity = hex_str(voter->identity_digest, + sizeof(voter->identity_digest)); + } else { + /* Consensus has multiple voters so no specific voter. */ + voter_identity = "consensus"; + } + + /* We extract both and on error, everything is stopped because it means + * the votes is malformed for the shared random value(s). */ + if (extract_one_srv(tokens, K_PREVIOUS_SRV, &ns->sr_info.previous_srv) < 0) { + log_warn(LD_DIR, "SR: Unable to parse previous SRV from %s", + voter_identity); + /* Maybe we have a chance with the current SRV so let's try it anyway. */ + } + if (extract_one_srv(tokens, K_CURRENT_SRV, &ns->sr_info.current_srv) < 0) { + log_warn(LD_DIR, "SR: Unable to parse current SRV from %s", + voter_identity); + } +} + /** Parse a v3 networkstatus vote, opinion, or consensus (depending on * ns_type), from <b>s</b>, and return the result. Return NULL on failure. */ networkstatus_t * @@ -3097,7 +3790,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, voter->nickname = tor_strdup(tok->args[0]); if (strlen(tok->args[1]) != HEX_DIGEST_LEN || base16_decode(voter->identity_digest, sizeof(voter->identity_digest), - tok->args[1], HEX_DIGEST_LEN) < 0) { + tok->args[1], HEX_DIGEST_LEN) + != sizeof(voter->identity_digest)) { log_warn(LD_DIR, "Error decoding identity digest %s in " "network-status document.", escaped(tok->args[1])); goto err; @@ -3146,7 +3840,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, } if (strlen(tok->args[0]) != HEX_DIGEST_LEN || base16_decode(voter->vote_digest, sizeof(voter->vote_digest), - tok->args[0], HEX_DIGEST_LEN) < 0) { + tok->args[0], HEX_DIGEST_LEN) + != sizeof(voter->vote_digest)) { log_warn(LD_DIR, "Error decoding vote digest %s in " "network-status consensus.", escaped(tok->args[0])); goto err; @@ -3171,7 +3866,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (strlen(tok->args[0]) == HEX_DIGEST_LEN) { networkstatus_voter_info_t *voter = smartlist_get(ns->voters, 0); if (base16_decode(voter->legacy_id_digest, DIGEST_LEN, - tok->args[0], HEX_DIGEST_LEN)<0) + tok->args[0], HEX_DIGEST_LEN) != DIGEST_LEN) bad = 1; else bad = 0; @@ -3182,6 +3877,22 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, } } + /* If this is a vote document, check if information about the shared + randomness protocol is included, and extract it. */ + if (ns->type == NS_TYPE_VOTE) { + /* Does this authority participates in the SR protocol? */ + tok = find_opt_by_keyword(tokens, K_SR_FLAG); + if (tok) { + ns->sr_info.participate = 1; + /* Get the SR commitments and reveals from the vote. */ + extract_shared_random_commits(ns, tokens); + } + } + /* For both a vote and consensus, extract the shared random values. */ + if (ns->type == NS_TYPE_VOTE || ns->type == NS_TYPE_CONSENSUS) { + extract_shared_random_srvs(ns, tokens); + } + /* Parse routerstatus lines. */ rs_tokens = smartlist_new(); rs_area = memarea_new(); @@ -3203,8 +3914,11 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, NULL, NULL, ns->consensus_method, - flav))) + flav))) { + /* Use exponential-backoff scheduling when downloading microdescs */ + rs->dl_status.backoff = DL_SCHED_RANDOM_EXPONENTIAL; smartlist_add(ns->routerstatus_list, rs); + } } } for (i = 1; i < smartlist_len(ns->routerstatus_list); ++i) { @@ -3330,7 +4044,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, if (strlen(id_hexdigest) != HEX_DIGEST_LEN || base16_decode(declared_identity, sizeof(declared_identity), - id_hexdigest, HEX_DIGEST_LEN) < 0) { + id_hexdigest, HEX_DIGEST_LEN) + != sizeof(declared_identity)) { log_warn(LD_DIR, "Error decoding declared identity %s in " "network-status document.", escaped(id_hexdigest)); goto err; @@ -3345,7 +4060,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, sig->alg = alg; if (strlen(sk_hexdigest) != HEX_DIGEST_LEN || base16_decode(sig->signing_key_digest, sizeof(sig->signing_key_digest), - sk_hexdigest, HEX_DIGEST_LEN) < 0) { + sk_hexdigest, HEX_DIGEST_LEN) + != sizeof(sig->signing_key_digest)) { log_warn(LD_DIR, "Error decoding declared signing key digest %s in " "network-status document.", escaped(sk_hexdigest)); tor_free(sig); @@ -3508,7 +4224,7 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) digest_algorithm_t alg; const char *flavor; const char *hexdigest; - size_t expected_length; + size_t expected_length, digest_length; tok = _tok; @@ -3531,8 +4247,8 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) continue; } - expected_length = - (alg == DIGEST_SHA1) ? HEX_DIGEST_LEN : HEX_DIGEST256_LEN; + digest_length = crypto_digest_algorithm_get_length(alg); + expected_length = digest_length * 2; /* hex encoding */ if (strlen(hexdigest) != expected_length) { log_warn(LD_DIR, "Wrong length on consensus-digest in detached " @@ -3541,13 +4257,13 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) } digests = detached_get_digests(sigs, flavor); tor_assert(digests); - if (!tor_mem_is_zero(digests->d[alg], DIGEST256_LEN)) { + if (!tor_mem_is_zero(digests->d[alg], digest_length)) { log_warn(LD_DIR, "Multiple digests for %s with %s on detached " "signatures document", flavor, algname); continue; } - if (base16_decode(digests->d[alg], DIGEST256_LEN, - hexdigest, strlen(hexdigest)) < 0) { + if (base16_decode(digests->d[alg], digest_length, + hexdigest, strlen(hexdigest)) != (int) digest_length) { log_warn(LD_DIR, "Bad encoding on consensus-digest in detached " "networkstatus signatures"); goto err; @@ -3620,14 +4336,14 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos) if (strlen(id_hexdigest) != HEX_DIGEST_LEN || base16_decode(id_digest, sizeof(id_digest), - id_hexdigest, HEX_DIGEST_LEN) < 0) { + id_hexdigest, HEX_DIGEST_LEN) != sizeof(id_digest)) { log_warn(LD_DIR, "Error decoding declared identity %s in " "network-status vote.", escaped(id_hexdigest)); goto err; } if (strlen(sk_hexdigest) != HEX_DIGEST_LEN || base16_decode(sk_digest, sizeof(sk_digest), - sk_hexdigest, HEX_DIGEST_LEN) < 0) { + sk_hexdigest, HEX_DIGEST_LEN) != sizeof(sk_digest)) { log_warn(LD_DIR, "Error decoding declared signing key digest %s in " "network-status vote.", escaped(sk_hexdigest)); goto err; @@ -4829,7 +5545,7 @@ tor_version_parse(const char *s, tor_version_t *out) memwipe(digest, 0, sizeof(digest)); if ( hexlen == 0 || (hexlen % 2) == 1) return -1; - if (base16_decode(digest, hexlen/2, cp, hexlen)) + if (base16_decode(digest, hexlen/2, cp, hexlen) != hexlen/2) return -1; memcpy(out->git_tag, digest, hexlen/2); out->git_tag_len = hexlen/2; @@ -4974,7 +5690,7 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out, eos = eos + 1; /* Check length. */ if (eos-desc > REND_DESC_MAX_SIZE) { - /* XXX023 If we are parsing this descriptor as a server, this + /* XXXX+ If we are parsing this descriptor as a server, this * should be a protocol warning. */ log_warn(LD_REND, "Descriptor length is %d which exceeds " "maximum rendezvous descriptor size of %d bytes.", @@ -5374,6 +6090,7 @@ rend_parse_client_keys(strmap_t *parsed_clients, const char *ckstr) directory_token_t *tok; const char *current_entry = NULL; memarea_t *area = NULL; + char *err_msg = NULL; if (!ckstr || strlen(ckstr) == 0) return -1; tokens = smartlist_new(); @@ -5383,8 +6100,6 @@ rend_parse_client_keys(strmap_t *parsed_clients, const char *ckstr) current_entry = eat_whitespace(ckstr); while (!strcmpstart(current_entry, "client-name ")) { rend_authorized_client_t *parsed_entry; - size_t len; - char descriptor_cookie_tmp[REND_DESC_COOKIE_LEN+2]; /* Determine end of string. */ const char *eos = strstr(current_entry, "\nclient-name "); if (!eos) @@ -5413,12 +6128,10 @@ rend_parse_client_keys(strmap_t *parsed_clients, const char *ckstr) tor_assert(tok == smartlist_get(tokens, 0)); tor_assert(tok->n_args == 1); - len = strlen(tok->args[0]); - if (len < 1 || len > 19 || - strspn(tok->args[0], REND_LEGAL_CLIENTNAME_CHARACTERS) != len) { + if (!rend_valid_client_name(tok->args[0])) { log_warn(LD_CONFIG, "Illegal client name: %s. (Length must be " - "between 1 and 19, and valid characters are " - "[A-Za-z0-9+-_].)", tok->args[0]); + "between 1 and %d, and valid characters are " + "[A-Za-z0-9+-_].)", tok->args[0], REND_CLIENTNAME_MAX_LEN); goto err; } /* Check if client name is duplicate. */ @@ -5440,23 +6153,13 @@ rend_parse_client_keys(strmap_t *parsed_clients, const char *ckstr) /* Parse descriptor cookie. */ tok = find_by_keyword(tokens, C_DESCRIPTOR_COOKIE); tor_assert(tok->n_args == 1); - if (strlen(tok->args[0]) != REND_DESC_COOKIE_LEN_BASE64 + 2) { - log_warn(LD_REND, "Descriptor cookie has illegal length: %s", - escaped(tok->args[0])); - goto err; - } - /* The size of descriptor_cookie_tmp needs to be REND_DESC_COOKIE_LEN+2, - * because a base64 encoding of length 24 does not fit into 16 bytes in all - * cases. */ - if (base64_decode(descriptor_cookie_tmp, sizeof(descriptor_cookie_tmp), - tok->args[0], strlen(tok->args[0])) - != REND_DESC_COOKIE_LEN) { - log_warn(LD_REND, "Descriptor cookie contains illegal characters: " - "%s", escaped(tok->args[0])); + if (rend_auth_decode_cookie(tok->args[0], parsed_entry->descriptor_cookie, + NULL, &err_msg) < 0) { + tor_assert(err_msg); + log_warn(LD_REND, "%s", err_msg); + tor_free(err_msg); goto err; } - memcpy(parsed_entry->descriptor_cookie, descriptor_cookie_tmp, - REND_DESC_COOKIE_LEN); } result = strmap_size(parsed_clients); goto done; @@ -5471,3 +6174,27 @@ rend_parse_client_keys(strmap_t *parsed_clients, const char *ckstr) return result; } +/** Called on startup; right now we just handle scanning the unparseable + * descriptor dumps, but hang anything else we might need to do in the + * future here as well. + */ +void +routerparse_init(void) +{ + /* + * Check both if the sandbox is active and whether it's configured; no + * point in loading all that if we won't be able to use it after the + * sandbox becomes active. + */ + if (!(sandbox_is_active() || get_options()->Sandbox)) { + dump_desc_init(); + } +} + +/** Clean up all data structures used by routerparse.c at exit */ +void +routerparse_free_all(void) +{ + dump_desc_fifo_cleanup(); +} + diff --git a/src/or/routerparse.h b/src/or/routerparse.h index c46eb1c0ae..81ef724945 100644 --- a/src/or/routerparse.h +++ b/src/or/routerparse.h @@ -85,11 +85,33 @@ int rend_parse_introduction_points(rend_service_descriptor_t *parsed, size_t intro_points_encoded_size); int rend_parse_client_keys(strmap_t *parsed_clients, const char *str); +void routerparse_init(void); +void routerparse_free_all(void); + #ifdef ROUTERPARSE_PRIVATE +/* + * One entry in the list of dumped descriptors; filename dumped to, length, + * SHA-256 and timestamp. + */ + +typedef struct { + char *filename; + size_t len; + uint8_t digest_sha256[DIGEST256_LEN]; + time_t when; +} dumped_desc_t; + +EXTERN(uint64_t, len_descs_dumped) +EXTERN(smartlist_t *, descs_dumped) STATIC int routerstatus_parse_guardfraction(const char *guardfraction_str, networkstatus_t *vote, vote_routerstatus_t *vote_rs, routerstatus_t *rs); +MOCK_DECL(STATIC dumped_desc_t *, dump_desc_populate_one_file, + (const char *dirname, const char *f)); +STATIC void dump_desc_populate_fifo_from_directory(const char *dirname); +STATIC void dump_desc(const char *desc, const char *type); +STATIC void dump_desc_fifo_cleanup(void); #endif #define ED_DESC_SIGNATURE_PREFIX "Tor router descriptor signature v1" diff --git a/src/or/scheduler.h b/src/or/scheduler.h index 94a44a0aa3..3dcfd2faca 100644 --- a/src/or/scheduler.h +++ b/src/or/scheduler.h @@ -44,6 +44,13 @@ MOCK_DECL(STATIC int, scheduler_compare_channels, (const void *c1_v, const void *c2_v)); STATIC uint64_t scheduler_get_queue_heuristic(void); STATIC void scheduler_update_queue_heuristic(time_t now); + +#ifdef TOR_UNIT_TESTS +extern smartlist_t *channels_pending; +extern struct event *run_sched_ev; +extern uint64_t queue_heuristic; +extern time_t queue_heuristic_timestamp; +#endif #endif #endif /* !defined(TOR_SCHEDULER_H) */ diff --git a/src/or/shared_random.c b/src/or/shared_random.c new file mode 100644 index 0000000000..c9886d92b0 --- /dev/null +++ b/src/or/shared_random.c @@ -0,0 +1,1363 @@ +/* Copyright (c) 2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file shared_random.c + * + * \brief Functions and data structure needed to accomplish the shared + * random protocol as defined in proposal #250. + * + * \details + * + * This file implements the dirauth-only commit-and-reveal protocol specified + * by proposal #250. The protocol has two phases (sr_phase_t): the commitment + * phase and the reveal phase (see get_sr_protocol_phase()). + * + * During the protocol, directory authorities keep state in memory (using + * sr_state_t) and in disk (using sr_disk_state_t). The synchronization between + * these two data structures happens in disk_state_update() and + * disk_state_parse(). + * + * Here is a rough protocol outline: + * + * 1) In the beginning of the commitment phase, dirauths generate a + * commitment/reveal value for the current protocol run (see + * new_protocol_run() and sr_generate_our_commit()). + * + * 2) During voting, dirauths publish their commits in their votes + * depending on the current phase. Dirauths also include the two + * latest shared random values (SRV) in their votes. + * (see sr_get_string_for_vote()) + * + * 3) Upon receiving a commit from a vote, authorities parse it, verify + * it, and attempt to save any new commitment or reveal information in + * their state file (see extract_shared_random_commits() and + * sr_handle_received_commits()). They also parse SRVs from votes to + * decide which SRV should be included in the final consensus (see + * extract_shared_random_srvs()). + * + * 3) After voting is done, we count the SRVs we extracted from the votes, + * to find the one voted by the majority of dirauths which should be + * included in the final consensus (see get_majority_srv_from_votes()). + * If an appropriate SRV is found, it is embedded in the consensus (see + * sr_get_string_for_consensus()). + * + * 4) At the end of the reveal phase, dirauths compute a fresh SRV for the + * day using the active commits (see sr_compute_srv()). This new SRV + * is embedded in the votes as described above. + * + * Some more notes: + * + * - To support rebooting authorities and to avoid double voting, each dirauth + * saves the current state of the protocol on disk so that it can resume + * normally in case of reboot. The disk state (sr_disk_state_t) is managed by + * shared_random_state.c:state_query() and we go to extra lengths to ensure + * that the state is flushed on disk everytime we receive any useful + * information like commits or SRVs. + * + * - When we receive a commit from a vote, we examine it to see if it's useful + * to us and whether it's appropriate to receive it according to the current + * phase of the protocol (see should_keep_commit()). If the commit is useful + * to us, we save it in our disk state using save_commit_to_state(). When we + * receive the reveal information corresponding to a commitment, we verify + * that they indeed match using verify_commit_and_reveal(). + * + * - We treat consensuses as the ground truth, so everytime we generate a new + * consensus we update our SR state accordingly even if our local view was + * different (see sr_act_post_consensus()). + * + * - After a consensus has been composed, the SR protocol state gets prepared + * for the next voting session using sr_state_update(). That function takes + * care of housekeeping and also rotates the SRVs and commits in case a new + * protocol run is coming up. We also call sr_state_update() on bootup (in + * sr_state_init()), to prepare the state for the very first voting session. + * + * Terminology: + * + * - "Commitment" is the commitment value of the commit-and-reveal protocol. + * + * - "Reveal" is the reveal value of the commit-and-reveal protocol. + * + * - "Commit" is a struct (sr_commit_t) that contains a commitment value and + * optionally also a corresponding reveal value. + * + * - "SRV" is the Shared Random Value that gets generated as the result of the + * commit-and-reveal protocol. + **/ + +#define SHARED_RANDOM_PRIVATE + +#include "or.h" +#include "shared_random.h" +#include "config.h" +#include "confparse.h" +#include "dirvote.h" +#include "networkstatus.h" +#include "routerkeys.h" +#include "router.h" +#include "routerlist.h" +#include "shared_random_state.h" +#include "util.h" + +/* String prefix of shared random values in votes/consensuses. */ +static const char previous_srv_str[] = "shared-rand-previous-value"; +static const char current_srv_str[] = "shared-rand-current-value"; +static const char commit_ns_str[] = "shared-rand-commit"; +static const char sr_flag_ns_str[] = "shared-rand-participate"; + +/* The value of the consensus param AuthDirNumSRVAgreements found in the + * vote. This is set once the consensus creation subsystem requests the + * SRV(s) that should be put in the consensus. We use this value to decide + * if we keep or not an SRV. */ +static int32_t num_srv_agreements_from_vote; + +/* Return a heap allocated copy of the SRV <b>orig</b>. */ +STATIC sr_srv_t * +srv_dup(const sr_srv_t *orig) +{ + sr_srv_t *dup = NULL; + + if (!orig) { + return NULL; + } + + dup = tor_malloc_zero(sizeof(sr_srv_t)); + dup->num_reveals = orig->num_reveals; + memcpy(dup->value, orig->value, sizeof(dup->value)); + return dup; +} + +/* Allocate a new commit object and initializing it with <b>rsa_identity</b> + * that MUST be provided. The digest algorithm is set to the default one + * that is supported. The rest is uninitialized. This never returns NULL. */ +static sr_commit_t * +commit_new(const char *rsa_identity) +{ + sr_commit_t *commit; + + tor_assert(rsa_identity); + + commit = tor_malloc_zero(sizeof(*commit)); + commit->alg = SR_DIGEST_ALG; + memcpy(commit->rsa_identity, rsa_identity, sizeof(commit->rsa_identity)); + base16_encode(commit->rsa_identity_hex, sizeof(commit->rsa_identity_hex), + commit->rsa_identity, sizeof(commit->rsa_identity)); + return commit; +} + +/* Issue a log message describing <b>commit</b>. */ +static void +commit_log(const sr_commit_t *commit) +{ + tor_assert(commit); + + log_debug(LD_DIR, "SR: Commit from %s", sr_commit_get_rsa_fpr(commit)); + log_debug(LD_DIR, "SR: Commit: [TS: %" PRIu64 "] [Encoded: %s]", + commit->commit_ts, commit->encoded_commit); + log_debug(LD_DIR, "SR: Reveal: [TS: %" PRIu64 "] [Encoded: %s]", + commit->reveal_ts, safe_str(commit->encoded_reveal)); +} + +/* Make sure that the commitment and reveal information in <b>commit</b> + * match. If they match return 0, return -1 otherwise. This function MUST be + * used everytime we receive a new reveal value. Furthermore, the commit + * object MUST have a reveal value and the hash of the reveal value. */ +STATIC int +verify_commit_and_reveal(const sr_commit_t *commit) +{ + tor_assert(commit); + + log_debug(LD_DIR, "SR: Validating commit from authority %s", + sr_commit_get_rsa_fpr(commit)); + + /* Check that the timestamps match. */ + if (commit->commit_ts != commit->reveal_ts) { + log_warn(LD_BUG, "SR: Commit timestamp %" PRIu64 " doesn't match reveal " + "timestamp %" PRIu64, commit->commit_ts, + commit->reveal_ts); + goto invalid; + } + + /* Verify that the hashed_reveal received in the COMMIT message, matches + * the reveal we just received. */ + { + /* We first hash the reveal we just received. */ + char received_hashed_reveal[sizeof(commit->hashed_reveal)]; + + /* Only sha3-256 is supported. */ + if (commit->alg != SR_DIGEST_ALG) { + goto invalid; + } + + /* Use the invariant length since the encoded reveal variable has an + * extra byte for the NUL terminated byte. */ + if (crypto_digest256(received_hashed_reveal, commit->encoded_reveal, + SR_REVEAL_BASE64_LEN, commit->alg)) { + /* Unable to digest the reveal blob, this is unlikely. */ + goto invalid; + } + + /* Now compare that with the hashed_reveal we received in COMMIT. */ + if (fast_memneq(received_hashed_reveal, commit->hashed_reveal, + sizeof(received_hashed_reveal))) { + log_warn(LD_BUG, "SR: Received reveal value from authority %s " + "does't match the commit value.", + sr_commit_get_rsa_fpr(commit)); + goto invalid; + } + } + + return 0; + invalid: + return -1; +} + +/* Return true iff the commit contains an encoded reveal value. */ +STATIC int +commit_has_reveal_value(const sr_commit_t *commit) +{ + return !tor_mem_is_zero(commit->encoded_reveal, + sizeof(commit->encoded_reveal)); +} + +/* Parse the encoded commit. The format is: + * base64-encode( TIMESTAMP || H(REVEAL) ) + * + * If successfully decoded and parsed, commit is updated and 0 is returned. + * On error, return -1. */ +STATIC int +commit_decode(const char *encoded, sr_commit_t *commit) +{ + int decoded_len = 0; + size_t offset = 0; + /* XXX: Needs two extra bytes for the base64 decode calculation matches + * the binary length once decoded. #17868. */ + char b64_decoded[SR_COMMIT_LEN + 2]; + + tor_assert(encoded); + tor_assert(commit); + + if (strlen(encoded) > SR_COMMIT_BASE64_LEN) { + /* This means that if we base64 decode successfully the reveiced commit, + * we'll end up with a bigger decoded commit thus unusable. */ + goto error; + } + + /* Decode our encoded commit. Let's be careful here since _encoded_ is + * coming from the network in a dirauth vote so we expect nothing more + * than the base64 encoded length of a commit. */ + decoded_len = base64_decode(b64_decoded, sizeof(b64_decoded), + encoded, strlen(encoded)); + if (decoded_len < 0) { + log_warn(LD_BUG, "SR: Commit from authority %s can't be decoded.", + sr_commit_get_rsa_fpr(commit)); + goto error; + } + + if (decoded_len != SR_COMMIT_LEN) { + log_warn(LD_BUG, "SR: Commit from authority %s decoded length doesn't " + "match the expected length (%d vs %u).", + sr_commit_get_rsa_fpr(commit), decoded_len, + (unsigned)SR_COMMIT_LEN); + goto error; + } + + /* First is the timestamp (8 bytes). */ + commit->commit_ts = tor_ntohll(get_uint64(b64_decoded)); + offset += sizeof(uint64_t); + /* Next is hashed reveal. */ + memcpy(commit->hashed_reveal, b64_decoded + offset, + sizeof(commit->hashed_reveal)); + /* Copy the base64 blob to the commit. Useful for voting. */ + strlcpy(commit->encoded_commit, encoded, sizeof(commit->encoded_commit)); + + return 0; + + error: + return -1; +} + +/* Parse the b64 blob at <b>encoded</b> containing reveal information and + * store the information in-place in <b>commit</b>. Return 0 on success else + * a negative value. */ +STATIC int +reveal_decode(const char *encoded, sr_commit_t *commit) +{ + int decoded_len = 0; + /* XXX: Needs two extra bytes for the base64 decode calculation matches + * the binary length once decoded. #17868. */ + char b64_decoded[SR_REVEAL_LEN + 2]; + + tor_assert(encoded); + tor_assert(commit); + + if (strlen(encoded) > SR_REVEAL_BASE64_LEN) { + /* This means that if we base64 decode successfully the received reveal + * value, we'll end up with a bigger decoded value thus unusable. */ + goto error; + } + + /* Decode our encoded reveal. Let's be careful here since _encoded_ is + * coming from the network in a dirauth vote so we expect nothing more + * than the base64 encoded length of our reveal. */ + decoded_len = base64_decode(b64_decoded, sizeof(b64_decoded), + encoded, strlen(encoded)); + if (decoded_len < 0) { + log_warn(LD_BUG, "SR: Reveal from authority %s can't be decoded.", + sr_commit_get_rsa_fpr(commit)); + goto error; + } + + if (decoded_len != SR_REVEAL_LEN) { + log_warn(LD_BUG, "SR: Reveal from authority %s decoded length is " + "doesn't match the expected length (%d vs %u)", + sr_commit_get_rsa_fpr(commit), decoded_len, + (unsigned)SR_REVEAL_LEN); + goto error; + } + + commit->reveal_ts = tor_ntohll(get_uint64(b64_decoded)); + /* Copy the last part, the random value. */ + memcpy(commit->random_number, b64_decoded + 8, + sizeof(commit->random_number)); + /* Also copy the whole message to use during verification */ + strlcpy(commit->encoded_reveal, encoded, sizeof(commit->encoded_reveal)); + + return 0; + + error: + return -1; +} + +/* Encode a reveal element using a given commit object to dst which is a + * buffer large enough to put the base64-encoded reveal construction. The + * format is as follow: + * REVEAL = base64-encode( TIMESTAMP || H(RN) ) + * Return base64 encoded length on success else a negative value. + */ +STATIC int +reveal_encode(const sr_commit_t *commit, char *dst, size_t len) +{ + int ret; + size_t offset = 0; + char buf[SR_REVEAL_LEN] = {0}; + + tor_assert(commit); + tor_assert(dst); + + set_uint64(buf, tor_htonll(commit->reveal_ts)); + offset += sizeof(uint64_t); + memcpy(buf + offset, commit->random_number, + sizeof(commit->random_number)); + + /* Let's clean the buffer and then b64 encode it. */ + memset(dst, 0, len); + ret = base64_encode(dst, len, buf, sizeof(buf), 0); + /* Wipe this buffer because it contains our random value. */ + memwipe(buf, 0, sizeof(buf)); + return ret; +} + +/* Encode the given commit object to dst which is a buffer large enough to + * put the base64-encoded commit. The format is as follow: + * COMMIT = base64-encode( TIMESTAMP || H(H(RN)) ) + * Return base64 encoded length on success else a negative value. + */ +STATIC int +commit_encode(const sr_commit_t *commit, char *dst, size_t len) +{ + size_t offset = 0; + char buf[SR_COMMIT_LEN] = {0}; + + tor_assert(commit); + tor_assert(dst); + + /* First is the timestamp (8 bytes). */ + set_uint64(buf, tor_htonll(commit->commit_ts)); + offset += sizeof(uint64_t); + /* and then the hashed reveal. */ + memcpy(buf + offset, commit->hashed_reveal, + sizeof(commit->hashed_reveal)); + + /* Clean the buffer and then b64 encode it. */ + memset(dst, 0, len); + return base64_encode(dst, len, buf, sizeof(buf), 0); +} + +/* Cleanup both our global state and disk state. */ +static void +sr_cleanup(void) +{ + sr_state_free(); +} + +/* Using <b>commit</b>, return a newly allocated string containing the commit + * information that should be used during SRV calculation. It's the caller + * responsibility to free the memory. Return NULL if this is not a commit to be + * used for SRV calculation. */ +static char * +get_srv_element_from_commit(const sr_commit_t *commit) +{ + char *element; + tor_assert(commit); + + if (!commit_has_reveal_value(commit)) { + return NULL; + } + + tor_asprintf(&element, "%s%s", sr_commit_get_rsa_fpr(commit), + commit->encoded_reveal); + return element; +} + +/* Return a srv object that is built with the construction: + * SRV = SHA3-256("shared-random" | INT_8(reveal_num) | + * INT_4(version) | HASHED_REVEALS | previous_SRV) + * This function cannot fail. */ +static sr_srv_t * +generate_srv(const char *hashed_reveals, uint64_t reveal_num, + const sr_srv_t *previous_srv) +{ + char msg[DIGEST256_LEN + SR_SRV_MSG_LEN] = {0}; + size_t offset = 0; + sr_srv_t *srv; + + tor_assert(hashed_reveals); + + /* Add the invariant token. */ + memcpy(msg, SR_SRV_TOKEN, SR_SRV_TOKEN_LEN); + offset += SR_SRV_TOKEN_LEN; + set_uint64(msg + offset, tor_htonll(reveal_num)); + offset += sizeof(uint64_t); + set_uint32(msg + offset, htonl(SR_PROTO_VERSION)); + offset += sizeof(uint32_t); + memcpy(msg + offset, hashed_reveals, DIGEST256_LEN); + offset += DIGEST256_LEN; + if (previous_srv != NULL) { + memcpy(msg + offset, previous_srv->value, sizeof(previous_srv->value)); + } + + /* Ok we have our message and key for the HMAC computation, allocate our + * srv object and do the last step. */ + srv = tor_malloc_zero(sizeof(*srv)); + crypto_digest256((char *) srv->value, msg, sizeof(msg), SR_DIGEST_ALG); + srv->num_reveals = reveal_num; + + { + /* Debugging. */ + char srv_hash_encoded[SR_SRV_VALUE_BASE64_LEN + 1]; + sr_srv_encode(srv_hash_encoded, sizeof(srv_hash_encoded), srv); + log_info(LD_DIR, "SR: Generated SRV: %s", srv_hash_encoded); + } + return srv; +} + +/* Compare reveal values and return the result. This should exclusively be + * used by smartlist_sort(). */ +static int +compare_reveal_(const void **_a, const void **_b) +{ + const sr_commit_t *a = *_a, *b = *_b; + return fast_memcmp(a->hashed_reveal, b->hashed_reveal, + sizeof(a->hashed_reveal)); +} + +/* Given <b>commit</b> give the line that we should place in our votes. + * It's the responsibility of the caller to free the string. */ +static char * +get_vote_line_from_commit(const sr_commit_t *commit, sr_phase_t phase) +{ + char *vote_line = NULL; + + switch (phase) { + case SR_PHASE_COMMIT: + tor_asprintf(&vote_line, "%s %u %s %s %s\n", + commit_ns_str, + SR_PROTO_VERSION, + crypto_digest_algorithm_get_name(commit->alg), + sr_commit_get_rsa_fpr(commit), + commit->encoded_commit); + break; + case SR_PHASE_REVEAL: + { + /* Send a reveal value for this commit if we have one. */ + const char *reveal_str = commit->encoded_reveal; + if (tor_mem_is_zero(commit->encoded_reveal, + sizeof(commit->encoded_reveal))) { + reveal_str = ""; + } + tor_asprintf(&vote_line, "%s %u %s %s %s %s\n", + commit_ns_str, + SR_PROTO_VERSION, + crypto_digest_algorithm_get_name(commit->alg), + sr_commit_get_rsa_fpr(commit), + commit->encoded_commit, reveal_str); + break; + } + default: + tor_assert(0); + } + + log_debug(LD_DIR, "SR: Commit vote line: %s", vote_line); + return vote_line; +} + +/* Return a heap allocated string that contains the given <b>srv</b> string + * representation formatted for a networkstatus document using the + * <b>key</b> as the start of the line. This doesn't return NULL. */ +static char * +srv_to_ns_string(const sr_srv_t *srv, const char *key) +{ + char *srv_str; + char srv_hash_encoded[SR_SRV_VALUE_BASE64_LEN + 1]; + tor_assert(srv); + tor_assert(key); + + sr_srv_encode(srv_hash_encoded, sizeof(srv_hash_encoded), srv); + tor_asprintf(&srv_str, "%s %" PRIu64 " %s\n", key, + srv->num_reveals, srv_hash_encoded); + log_debug(LD_DIR, "SR: Consensus SRV line: %s", srv_str); + return srv_str; +} + +/* Given the previous SRV and the current SRV, return a heap allocated + * string with their data that could be put in a vote or a consensus. Caller + * must free the returned string. Return NULL if no SRVs were provided. */ +static char * +get_ns_str_from_sr_values(const sr_srv_t *prev_srv, const sr_srv_t *cur_srv) +{ + smartlist_t *chunks = NULL; + char *srv_str; + + if (!prev_srv && !cur_srv) { + return NULL; + } + + chunks = smartlist_new(); + + if (prev_srv) { + char *srv_line = srv_to_ns_string(prev_srv, previous_srv_str); + smartlist_add(chunks, srv_line); + } + + if (cur_srv) { + char *srv_line = srv_to_ns_string(cur_srv, current_srv_str); + smartlist_add(chunks, srv_line); + } + + /* Join the line(s) here in one string to return. */ + srv_str = smartlist_join_strings(chunks, "", 0, NULL); + SMARTLIST_FOREACH(chunks, char *, s, tor_free(s)); + smartlist_free(chunks); + + return srv_str; +} + +/* Return 1 iff the two commits have the same commitment values. This + * function does not care about reveal values. */ +STATIC int +commitments_are_the_same(const sr_commit_t *commit_one, + const sr_commit_t *commit_two) +{ + tor_assert(commit_one); + tor_assert(commit_two); + + if (strcmp(commit_one->encoded_commit, commit_two->encoded_commit)) { + return 0; + } + return 1; +} + +/* We just received a commit from the vote of authority with + * <b>identity_digest</b>. Return 1 if this commit is authorititative that + * is, it belongs to the authority that voted it. Else return 0 if not. */ +STATIC int +commit_is_authoritative(const sr_commit_t *commit, + const char *voter_key) +{ + tor_assert(commit); + tor_assert(voter_key); + + return !memcmp(commit->rsa_identity, voter_key, + sizeof(commit->rsa_identity)); +} + +/* Decide if the newly received <b>commit</b> should be kept depending on + * the current phase and state of the protocol. The <b>voter_key</b> is the + * RSA identity key fingerprint of the authority's vote from which the + * commit comes from. The <b>phase</b> is the phase we should be validating + * the commit for. Return 1 if the commit should be added to our state or 0 + * if not. */ +STATIC int +should_keep_commit(const sr_commit_t *commit, const char *voter_key, + sr_phase_t phase) +{ + const sr_commit_t *saved_commit; + + tor_assert(commit); + tor_assert(voter_key); + + log_debug(LD_DIR, "SR: Inspecting commit from %s (voter: %s)?", + sr_commit_get_rsa_fpr(commit), + hex_str(voter_key, DIGEST_LEN)); + + /* For a commit to be considered, it needs to be authoritative (it should + * be the voter's own commit). */ + if (!commit_is_authoritative(commit, voter_key)) { + log_debug(LD_DIR, "SR: Ignoring non-authoritative commit."); + goto ignore; + } + + /* Let's make sure, for extra safety, that this fingerprint is known to + * us. Even though this comes from a vote, doesn't hurt to be + * extracareful. */ + if (trusteddirserver_get_by_v3_auth_digest(commit->rsa_identity) == NULL) { + log_warn(LD_DIR, "SR: Fingerprint %s is not from a recognized " + "authority. Discarding commit.", + escaped(commit->rsa_identity)); + goto ignore; + } + + /* Check if the authority that voted for <b>commit</b> has already posted + * a commit before. */ + saved_commit = sr_state_get_commit(commit->rsa_identity); + + switch (phase) { + case SR_PHASE_COMMIT: + /* Already having a commit for an authority so ignore this one. */ + if (saved_commit) { + /* Receiving known commits should happen naturally since commit phase + lasts multiple rounds. However if the commitment value changes + during commit phase, it might be a bug so log more loudly. */ + if (!commitments_are_the_same(commit, saved_commit)) { + log_info(LD_DIR, + "SR: Received altered commit from %s in commit phase.", + sr_commit_get_rsa_fpr(commit)); + } else { + log_debug(LD_DIR, "SR: Ignoring known commit during commit phase."); + } + goto ignore; + } + + /* A commit with a reveal value during commitment phase is very wrong. */ + if (commit_has_reveal_value(commit)) { + log_warn(LD_DIR, "SR: Commit from authority %s has a reveal value " + "during COMMIT phase. (voter: %s)", + sr_commit_get_rsa_fpr(commit), + hex_str(voter_key, DIGEST_LEN)); + goto ignore; + } + break; + case SR_PHASE_REVEAL: + /* We are now in reveal phase. We keep a commit if and only if: + * + * - We have already seen a commit by this auth, AND + * - the saved commit has the same commitment value as this one, AND + * - the saved commit has no reveal information, AND + * - this commit does have reveal information, AND + * - the reveal & commit information are matching. + * + * If all the above are true, then we are interested in this new commit + * for its reveal information. */ + + if (!saved_commit) { + log_debug(LD_DIR, "SR: Ignoring commit first seen in reveal phase."); + goto ignore; + } + + if (!commitments_are_the_same(commit, saved_commit)) { + log_warn(LD_DIR, "SR: Commit from authority %s is different from " + "previous commit in our state (voter: %s)", + sr_commit_get_rsa_fpr(commit), + hex_str(voter_key, DIGEST_LEN)); + goto ignore; + } + + if (commit_has_reveal_value(saved_commit)) { + log_debug(LD_DIR, "SR: Ignoring commit with known reveal info."); + goto ignore; + } + + if (!commit_has_reveal_value(commit)) { + log_debug(LD_DIR, "SR: Ignoring commit without reveal value."); + goto ignore; + } + + if (verify_commit_and_reveal(commit) < 0) { + log_warn(LD_BUG, "SR: Commit from authority %s has an invalid " + "reveal value. (voter: %s)", + sr_commit_get_rsa_fpr(commit), + hex_str(voter_key, DIGEST_LEN)); + goto ignore; + } + break; + default: + tor_assert(0); + } + + return 1; + + ignore: + return 0; +} + +/* We are in reveal phase and we found a valid and verified <b>commit</b> in + * a vote that contains reveal values that we could use. Update the commit + * we have in our state. Never call this with an unverified commit. */ +STATIC void +save_commit_during_reveal_phase(const sr_commit_t *commit) +{ + sr_commit_t *saved_commit; + + tor_assert(commit); + + /* Get the commit from our state. */ + saved_commit = sr_state_get_commit(commit->rsa_identity); + tor_assert(saved_commit); + /* Safety net. They can not be different commitments at this point. */ + int same_commits = commitments_are_the_same(commit, saved_commit); + tor_assert(same_commits); + + /* Copy reveal information to our saved commit. */ + sr_state_copy_reveal_info(saved_commit, commit); +} + +/* Save <b>commit</b> to our persistent state. Depending on the current + * phase, different actions are taken. Steals reference of <b>commit</b>. + * The commit object MUST be valid and verified before adding it to the + * state. */ +STATIC void +save_commit_to_state(sr_commit_t *commit) +{ + sr_phase_t phase = sr_state_get_phase(); + + ASSERT_COMMIT_VALID(commit); + + switch (phase) { + case SR_PHASE_COMMIT: + /* During commit phase, just save any new authoritative commit */ + sr_state_add_commit(commit); + break; + case SR_PHASE_REVEAL: + save_commit_during_reveal_phase(commit); + sr_commit_free(commit); + break; + default: + tor_assert(0); + } +} + +/* Return 1 if we should we keep an SRV voted by <b>n_agreements</b> auths. + * Return 0 if we should ignore it. */ +static int +should_keep_srv(int n_agreements) +{ + /* Check if the most popular SRV has reached majority. */ + int n_voters = get_n_authorities(V3_DIRINFO); + int votes_required_for_majority = (n_voters / 2) + 1; + + /* We need at the very least majority to keep a value. */ + if (n_agreements < votes_required_for_majority) { + log_notice(LD_DIR, "SR: SRV didn't reach majority [%d/%d]!", + n_agreements, votes_required_for_majority); + return 0; + } + + /* When we just computed a new SRV, we need to have super majority in order + * to keep it. */ + if (sr_state_srv_is_fresh()) { + /* Check if we have super majority for this new SRV value. */ + if (n_agreements < num_srv_agreements_from_vote) { + log_notice(LD_DIR, "SR: New SRV didn't reach agreement [%d/%d]!", + n_agreements, num_srv_agreements_from_vote); + return 0; + } + } + + return 1; +} + +/* Helper: compare two DIGEST256_LEN digests. */ +static int +compare_srvs_(const void **_a, const void **_b) +{ + const sr_srv_t *a = *_a, *b = *_b; + return tor_memcmp(a->value, b->value, sizeof(a->value)); +} + +/* Return the most frequent member of the sorted list of DIGEST256_LEN + * digests in <b>sl</b> with the count of that most frequent element. */ +static sr_srv_t * +smartlist_get_most_frequent_srv(const smartlist_t *sl, int *count_out) +{ + return smartlist_get_most_frequent_(sl, compare_srvs_, count_out); +} + +/** Compare two SRVs. Used in smartlist sorting. */ +static int +compare_srv_(const void **_a, const void **_b) +{ + const sr_srv_t *a = *_a, *b = *_b; + return fast_memcmp(a->value, b->value, + sizeof(a->value)); +} + +/* Using a list of <b>votes</b>, return the SRV object from them that has + * been voted by the majority of dirauths. If <b>current</b> is set, we look + * for the current SRV value else the previous one. The returned pointer is + * an object located inside a vote. NULL is returned if no appropriate value + * could be found. */ +STATIC sr_srv_t * +get_majority_srv_from_votes(const smartlist_t *votes, int current) +{ + int count = 0; + sr_srv_t *most_frequent_srv = NULL; + sr_srv_t *the_srv = NULL; + smartlist_t *srv_list; + + tor_assert(votes); + + srv_list = smartlist_new(); + + /* Walk over votes and register any SRVs found. */ + SMARTLIST_FOREACH_BEGIN(votes, networkstatus_t *, v) { + sr_srv_t *srv_tmp = NULL; + + if (!v->sr_info.participate) { + /* Ignore vote that do not participate. */ + continue; + } + /* Do we want previous or current SRV? */ + srv_tmp = current ? v->sr_info.current_srv : v->sr_info.previous_srv; + if (!srv_tmp) { + continue; + } + + smartlist_add(srv_list, srv_tmp); + } SMARTLIST_FOREACH_END(v); + + smartlist_sort(srv_list, compare_srv_); + most_frequent_srv = smartlist_get_most_frequent_srv(srv_list, &count); + if (!most_frequent_srv) { + goto end; + } + + /* Was this SRV voted by enough auths for us to keep it? */ + if (!should_keep_srv(count)) { + goto end; + } + + /* We found an SRV that we can use! Habemus SRV! */ + the_srv = most_frequent_srv; + + { + /* Debugging */ + char encoded[SR_SRV_VALUE_BASE64_LEN + 1]; + sr_srv_encode(encoded, sizeof(encoded), the_srv); + log_debug(LD_DIR, "SR: Chosen SRV by majority: %s (%d votes)", encoded, + count); + } + + end: + /* We do not free any sr_srv_t values, we don't have the ownership. */ + smartlist_free(srv_list); + return the_srv; +} + +/* Encode the given shared random value and put it in dst. Destination + * buffer must be at least SR_SRV_VALUE_BASE64_LEN plus the NULL byte. */ +void +sr_srv_encode(char *dst, size_t dst_len, const sr_srv_t *srv) +{ + int ret; + /* Extra byte for the NULL terminated char. */ + char buf[SR_SRV_VALUE_BASE64_LEN + 1]; + + tor_assert(dst); + tor_assert(srv); + tor_assert(dst_len >= sizeof(buf)); + + ret = base64_encode(buf, sizeof(buf), (const char *) srv->value, + sizeof(srv->value), 0); + /* Always expect the full length without the NULL byte. */ + tor_assert(ret == (sizeof(buf) - 1)); + tor_assert(ret <= (int) dst_len); + strlcpy(dst, buf, dst_len); +} + +/* Free a commit object. */ +void +sr_commit_free(sr_commit_t *commit) +{ + if (commit == NULL) { + return; + } + /* Make sure we do not leave OUR random number in memory. */ + memwipe(commit->random_number, 0, sizeof(commit->random_number)); + tor_free(commit); +} + +/* Generate the commitment/reveal value for the protocol run starting at + * <b>timestamp</b>. <b>my_rsa_cert</b> is our authority RSA certificate. */ +sr_commit_t * +sr_generate_our_commit(time_t timestamp, const authority_cert_t *my_rsa_cert) +{ + sr_commit_t *commit = NULL; + char digest[DIGEST_LEN]; + + tor_assert(my_rsa_cert); + + /* Get our RSA identity fingerprint */ + if (crypto_pk_get_digest(my_rsa_cert->identity_key, digest) < 0) { + goto error; + } + + /* New commit with our identity key. */ + commit = commit_new(digest); + + /* Generate the reveal random value */ + crypto_strongest_rand(commit->random_number, + sizeof(commit->random_number)); + commit->commit_ts = commit->reveal_ts = timestamp; + + /* Now get the base64 blob that corresponds to our reveal */ + if (reveal_encode(commit, commit->encoded_reveal, + sizeof(commit->encoded_reveal)) < 0) { + log_err(LD_DIR, "SR: Unable to encode our reveal value!"); + goto error; + } + + /* Now let's create the commitment */ + tor_assert(commit->alg == SR_DIGEST_ALG); + /* The invariant length is used here since the encoded reveal variable + * has an extra byte added for the NULL terminated byte. */ + if (crypto_digest256(commit->hashed_reveal, commit->encoded_reveal, + SR_REVEAL_BASE64_LEN, commit->alg)) { + goto error; + } + + /* Now get the base64 blob that corresponds to our commit. */ + if (commit_encode(commit, commit->encoded_commit, + sizeof(commit->encoded_commit)) < 0) { + log_err(LD_DIR, "SR: Unable to encode our commit value!"); + goto error; + } + + log_debug(LD_DIR, "SR: Generated our commitment:"); + commit_log(commit); + /* Our commit better be valid :). */ + commit->valid = 1; + return commit; + + error: + sr_commit_free(commit); + return NULL; +} + +/* Compute the shared random value based on the active commits in our state. */ +void +sr_compute_srv(void) +{ + uint64_t reveal_num = 0; + char *reveals = NULL; + smartlist_t *chunks, *commits; + digestmap_t *state_commits; + + /* Computing a shared random value in the commit phase is very wrong. This + * should only happen at the very end of the reveal phase when a new + * protocol run is about to start. */ + tor_assert(sr_state_get_phase() == SR_PHASE_REVEAL); + state_commits = sr_state_get_commits(); + + commits = smartlist_new(); + chunks = smartlist_new(); + + /* We must make a list of commit ordered by authority fingerprint in + * ascending order as specified by proposal 250. */ + DIGESTMAP_FOREACH(state_commits, key, sr_commit_t *, c) { + /* Extra safety net, make sure we have valid commit before using it. */ + ASSERT_COMMIT_VALID(c); + /* Let's not use a commit from an authority that we don't know. It's + * possible that an authority could be removed during a protocol run so + * that commit value should never be used in the SRV computation. */ + if (trusteddirserver_get_by_v3_auth_digest(c->rsa_identity) == NULL) { + log_warn(LD_DIR, "SR: Fingerprint %s is not from a recognized " + "authority. Discarding commit for the SRV computation.", + sr_commit_get_rsa_fpr(c)); + continue; + } + /* We consider this commit valid. */ + smartlist_add(commits, c); + } DIGESTMAP_FOREACH_END; + smartlist_sort(commits, compare_reveal_); + + /* Now for each commit for that sorted list in ascending order, we'll + * build the element for each authority that needs to go into the srv + * computation. */ + SMARTLIST_FOREACH_BEGIN(commits, const sr_commit_t *, c) { + char *element = get_srv_element_from_commit(c); + if (element) { + smartlist_add(chunks, element); + reveal_num++; + } + } SMARTLIST_FOREACH_END(c); + smartlist_free(commits); + + { + /* Join all reveal values into one giant string that we'll hash so we + * can generated our shared random value. */ + sr_srv_t *current_srv; + char hashed_reveals[DIGEST256_LEN]; + reveals = smartlist_join_strings(chunks, "", 0, NULL); + SMARTLIST_FOREACH(chunks, char *, s, tor_free(s)); + smartlist_free(chunks); + if (crypto_digest256(hashed_reveals, reveals, strlen(reveals), + SR_DIGEST_ALG)) { + goto end; + } + current_srv = generate_srv(hashed_reveals, reveal_num, + sr_state_get_previous_srv()); + sr_state_set_current_srv(current_srv); + /* We have a fresh SRV, flag our state. */ + sr_state_set_fresh_srv(); + } + + end: + tor_free(reveals); +} + +/* Parse a list of arguments from a SRV value either from a vote, consensus + * or from our disk state and return a newly allocated srv object. NULL is + * returned on error. + * + * The arguments' order: + * num_reveals, value + */ +sr_srv_t * +sr_parse_srv(const smartlist_t *args) +{ + char *value; + int ok, ret; + uint64_t num_reveals; + sr_srv_t *srv = NULL; + + tor_assert(args); + + if (smartlist_len(args) < 2) { + goto end; + } + + /* First argument is the number of reveal values */ + num_reveals = tor_parse_uint64(smartlist_get(args, 0), + 10, 0, UINT64_MAX, &ok, NULL); + if (!ok) { + goto end; + } + /* Second and last argument is the shared random value it self. */ + value = smartlist_get(args, 1); + if (strlen(value) != SR_SRV_VALUE_BASE64_LEN) { + goto end; + } + + srv = tor_malloc_zero(sizeof(*srv)); + srv->num_reveals = num_reveals; + /* We substract one byte from the srclen because the function ignores the + * '=' character in the given buffer. This is broken but it's a documented + * behavior of the implementation. */ + ret = base64_decode((char *) srv->value, sizeof(srv->value), value, + SR_SRV_VALUE_BASE64_LEN - 1); + if (ret != sizeof(srv->value)) { + tor_free(srv); + srv = NULL; + goto end; + } + end: + return srv; +} + +/* Parse a commit from a vote or from our disk state and return a newly + * allocated commit object. NULL is returned on error. + * + * The commit's data is in <b>args</b> and the order matters very much: + * version, algname, RSA fingerprint, commit value[, reveal value] + */ +sr_commit_t * +sr_parse_commit(const smartlist_t *args) +{ + uint32_t version; + char *value, digest[DIGEST_LEN]; + digest_algorithm_t alg; + const char *rsa_identity_fpr; + sr_commit_t *commit = NULL; + + if (smartlist_len(args) < 4) { + goto error; + } + + /* First is the version number of the SR protocol which indicates at which + * version that commit was created. */ + value = smartlist_get(args, 0); + version = (uint32_t) tor_parse_ulong(value, 10, 1, UINT32_MAX, NULL, NULL); + if (version > SR_PROTO_VERSION) { + log_info(LD_DIR, "SR: Commit version %" PRIu32 " (%s) is not supported.", + version, escaped(value)); + goto error; + } + + /* Second is the algorithm. */ + value = smartlist_get(args, 1); + alg = crypto_digest_algorithm_parse_name(value); + if (alg != SR_DIGEST_ALG) { + log_warn(LD_BUG, "SR: Commit algorithm %s is not recognized.", + escaped(value)); + goto error; + } + + /* Third argument is the RSA fingerprint of the auth and turn it into a + * digest value. */ + rsa_identity_fpr = smartlist_get(args, 2); + if (base16_decode(digest, DIGEST_LEN, rsa_identity_fpr, + HEX_DIGEST_LEN) < 0) { + log_warn(LD_DIR, "SR: RSA fingerprint %s not decodable", + escaped(rsa_identity_fpr)); + goto error; + } + + /* Allocate commit since we have a valid identity now. */ + commit = commit_new(digest); + + /* Fourth argument is the commitment value base64-encoded. */ + value = smartlist_get(args, 3); + if (commit_decode(value, commit) < 0) { + goto error; + } + + /* (Optional) Fifth argument is the revealed value. */ + if (smartlist_len(args) > 4) { + value = smartlist_get(args, 4); + if (reveal_decode(value, commit) < 0) { + goto error; + } + } + + return commit; + + error: + sr_commit_free(commit); + return NULL; +} + +/* Called when we are done parsing a vote by <b>voter_key</b> that might + * contain some useful <b>commits</b>. Find if any of them should be kept + * and update our state accordingly. Once done, the list of commitments will + * be empty. */ +void +sr_handle_received_commits(smartlist_t *commits, crypto_pk_t *voter_key) +{ + char rsa_identity[DIGEST_LEN]; + + tor_assert(voter_key); + + /* It's possible that the vote has _NO_ commits. */ + if (commits == NULL) { + return; + } + + /* Get the RSA identity fingerprint of this voter */ + if (crypto_pk_get_digest(voter_key, rsa_identity) < 0) { + return; + } + + SMARTLIST_FOREACH_BEGIN(commits, sr_commit_t *, commit) { + /* We won't need the commit in this list anymore, kept or not. */ + SMARTLIST_DEL_CURRENT(commits, commit); + /* Check if this commit is valid and should be stored in our state. */ + if (!should_keep_commit(commit, rsa_identity, + sr_state_get_phase())) { + sr_commit_free(commit); + continue; + } + /* Ok, we have a valid commit now that we are about to put in our state. + * so flag it valid from now on. */ + commit->valid = 1; + /* Everything lines up: save this commit to state then! */ + save_commit_to_state(commit); + } SMARTLIST_FOREACH_END(commit); +} + +/* Return a heap-allocated string containing commits that should be put in + * the votes. It's the responsibility of the caller to free the string. + * This always return a valid string, either empty or with line(s). */ +char * +sr_get_string_for_vote(void) +{ + char *vote_str = NULL; + digestmap_t *state_commits; + smartlist_t *chunks = smartlist_new(); + const or_options_t *options = get_options(); + + /* Are we participating in the protocol? */ + if (!options->AuthDirSharedRandomness) { + goto end; + } + + log_debug(LD_DIR, "SR: Preparing our vote info:"); + + /* First line, put in the vote the participation flag. */ + { + char *sr_flag_line; + tor_asprintf(&sr_flag_line, "%s\n", sr_flag_ns_str); + smartlist_add(chunks, sr_flag_line); + } + + /* In our vote we include every commitment in our permanent state. */ + state_commits = sr_state_get_commits(); + smartlist_t *state_commit_vote_lines = smartlist_new(); + DIGESTMAP_FOREACH(state_commits, key, const sr_commit_t *, commit) { + char *line = get_vote_line_from_commit(commit, sr_state_get_phase()); + smartlist_add(state_commit_vote_lines, line); + } DIGESTMAP_FOREACH_END; + + /* Sort the commit strings by version (string, not numeric), algorithm, + * and fingerprint. This makes sure the commit lines in votes are in a + * recognisable, stable order. */ + smartlist_sort_strings(state_commit_vote_lines); + + /* Now add the sorted list of commits to the vote */ + smartlist_add_all(chunks, state_commit_vote_lines); + smartlist_free(state_commit_vote_lines); + + /* Add the SRV value(s) if any. */ + { + char *srv_lines = get_ns_str_from_sr_values(sr_state_get_previous_srv(), + sr_state_get_current_srv()); + if (srv_lines) { + smartlist_add(chunks, srv_lines); + } + } + + end: + vote_str = smartlist_join_strings(chunks, "", 0, NULL); + SMARTLIST_FOREACH(chunks, char *, s, tor_free(s)); + smartlist_free(chunks); + return vote_str; +} + +/* Return a heap-allocated string that should be put in the consensus and + * contains the shared randomness values. It's the responsibility of the + * caller to free the string. NULL is returned if no SRV(s) available. + * + * This is called when a consensus (any flavor) is bring created thus it + * should NEVER change the state nor the state should be changed in between + * consensus creation. + * + * <b>num_srv_agreements</b> is taken from the votes thus the voted value + * that should be used. + * */ +char * +sr_get_string_for_consensus(const smartlist_t *votes, + int32_t num_srv_agreements) +{ + char *srv_str; + const or_options_t *options = get_options(); + + tor_assert(votes); + + /* Not participating, avoid returning anything. */ + if (!options->AuthDirSharedRandomness) { + log_info(LD_DIR, "SR: Support disabled (AuthDirSharedRandomness %d)", + options->AuthDirSharedRandomness); + goto end; + } + + /* Set the global value of AuthDirNumSRVAgreements found in the votes. */ + num_srv_agreements_from_vote = num_srv_agreements; + + /* Check the votes and figure out if SRVs should be included in the final + * consensus. */ + sr_srv_t *prev_srv = get_majority_srv_from_votes(votes, 0); + sr_srv_t *cur_srv = get_majority_srv_from_votes(votes, 1); + srv_str = get_ns_str_from_sr_values(prev_srv, cur_srv); + if (!srv_str) { + goto end; + } + + return srv_str; + end: + return NULL; +} + +/* We just computed a new <b>consensus</b>. Update our state with the SRVs + * from the consensus (might be NULL as well). Register the SRVs in our SR + * state and prepare for the upcoming protocol round. */ +void +sr_act_post_consensus(const networkstatus_t *consensus) +{ + const or_options_t *options = get_options(); + + /* Don't act if our state hasn't been initialized. We can be called during + * boot time when loading consensus from disk which is prior to the + * initialization of the SR subsystem. We also should not be doing + * anything if we are _not_ a directory authority and if we are a bridge + * authority. */ + if (!sr_state_is_initialized() || !authdir_mode_v3(options) || + authdir_mode_bridge(options)) { + return; + } + + /* Set the majority voted SRVs in our state even if both are NULL. It + * doesn't matter this is what the majority has decided. Obviously, we can + * only do that if we have a consensus. */ + if (consensus) { + /* Start by freeing the current SRVs since the SRVs we believed during + * voting do not really matter. Now that all the votes are in, we use the + * majority's opinion on which are the active SRVs. */ + sr_state_clean_srvs(); + /* Reset the fresh flag of the SRV so we know that from now on we don't + * have a new SRV to vote for. We just used the one from the consensus + * decided by the majority. */ + sr_state_unset_fresh_srv(); + /* Set the SR values from the given consensus. */ + sr_state_set_previous_srv(srv_dup(consensus->sr_info.previous_srv)); + sr_state_set_current_srv(srv_dup(consensus->sr_info.current_srv)); + } + + /* Prepare our state so that it's ready for the next voting period. */ + { + voting_schedule_t *voting_schedule = + get_voting_schedule(options,time(NULL), LOG_NOTICE); + time_t interval_starts = voting_schedule->interval_starts; + sr_state_update(interval_starts); + tor_free(voting_schedule); + } +} + +/* Initialize shared random subsystem. This MUST be called early in the boot + * process of tor. Return 0 on success else -1 on error. */ +int +sr_init(int save_to_disk) +{ + return sr_state_init(save_to_disk, 1); +} + +/* Save our state to disk and cleanup everything. */ +void +sr_save_and_cleanup(void) +{ + sr_state_save(); + sr_cleanup(); +} + +#ifdef TOR_UNIT_TESTS + +/* Set the global value of number of SRV agreements so the test can play + * along by calling specific functions that don't parse the votes prior for + * the AuthDirNumSRVAgreements value. */ +void +set_num_srv_agreements(int32_t value) +{ + num_srv_agreements_from_vote = value; +} + +#endif /* TOR_UNIT_TESTS */ + diff --git a/src/or/shared_random.h b/src/or/shared_random.h new file mode 100644 index 0000000000..9885934cc7 --- /dev/null +++ b/src/or/shared_random.h @@ -0,0 +1,168 @@ +/* Copyright (c) 2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_SHARED_RANDOM_H +#define TOR_SHARED_RANDOM_H + +/* + * This file contains ABI/API of the shared random protocol defined in + * proposal #250. Every public functions and data structure are namespaced + * with "sr_" which stands for shared random. + */ + +#include "or.h" + +/* Protocol version */ +#define SR_PROTO_VERSION 1 +/* Default digest algorithm. */ +#define SR_DIGEST_ALG DIGEST_SHA3_256 +/* Invariant token in the SRV calculation. */ +#define SR_SRV_TOKEN "shared-random" +/* Don't count the NUL terminated byte even though the TOKEN has it. */ +#define SR_SRV_TOKEN_LEN (sizeof(SR_SRV_TOKEN) - 1) + +/* Length of the random number (in bytes). */ +#define SR_RANDOM_NUMBER_LEN 32 +/* Size of a decoded commit value in a vote or state. It's a hash and a + * timestamp. It adds up to 40 bytes. */ +#define SR_COMMIT_LEN (sizeof(uint64_t) + DIGEST256_LEN) +/* Size of a decoded reveal value from a vote or state. It's a 64 bit + * timestamp and the hashed random number. This adds up to 40 bytes. */ +#define SR_REVEAL_LEN (sizeof(uint64_t) + DIGEST256_LEN) +/* Size of SRV message length. The construction is has follow: + * "shared-random" | INT_8(reveal_num) | INT_4(version) | PREV_SRV */ +#define SR_SRV_MSG_LEN \ + (SR_SRV_TOKEN_LEN + sizeof(uint64_t) + sizeof(uint32_t) + DIGEST256_LEN) + +/* Length of base64 encoded commit NOT including the NUL terminated byte. + * Formula is taken from base64_encode_size. This adds up to 56 bytes. */ +#define SR_COMMIT_BASE64_LEN \ + (((SR_COMMIT_LEN - 1) / 3) * 4 + 4) +/* Length of base64 encoded reveal NOT including the NUL terminated byte. + * Formula is taken from base64_encode_size. This adds up to 56 bytes. */ +#define SR_REVEAL_BASE64_LEN \ + (((SR_REVEAL_LEN - 1) / 3) * 4 + 4) +/* Length of base64 encoded shared random value. It's 32 bytes long so 44 + * bytes from the base64_encode_size formula. That includes the '=' + * character at the end. */ +#define SR_SRV_VALUE_BASE64_LEN \ + (((DIGEST256_LEN - 1) / 3) * 4 + 4) + +/* Assert if commit valid flag is not set. */ +#define ASSERT_COMMIT_VALID(c) tor_assert((c)->valid) + +/* Protocol phase. */ +typedef enum { + /* Commitment phase */ + SR_PHASE_COMMIT = 1, + /* Reveal phase */ + SR_PHASE_REVEAL = 2, +} sr_phase_t; + +/* A shared random value (SRV). */ +typedef struct sr_srv_t { + /* The number of reveal values used to derive this SRV. */ + uint64_t num_reveals; + /* The actual value. This is the stored result of SHA3-256. */ + uint8_t value[DIGEST256_LEN]; +} sr_srv_t; + +/* A commit (either ours or from another authority). */ +typedef struct sr_commit_t { + /* Hashing algorithm used. */ + digest_algorithm_t alg; + /* Indicate if this commit has been verified thus valid. */ + unsigned int valid:1; + + /* Commit owner info */ + + /* The RSA identity key of the authority and its base16 representation, + * which includes the NUL terminated byte. */ + char rsa_identity[DIGEST_LEN]; + char rsa_identity_hex[HEX_DIGEST_LEN + 1]; + + /* Commitment information */ + + /* Timestamp of reveal. Correspond to TIMESTAMP. */ + uint64_t reveal_ts; + /* H(REVEAL) as found in COMMIT message. */ + char hashed_reveal[DIGEST256_LEN]; + /* Base64 encoded COMMIT. We use this to put it in our vote. */ + char encoded_commit[SR_COMMIT_BASE64_LEN + 1]; + + /* Reveal information */ + + /* H(RN) which is what we used as the random value for this commit. We + * don't use the raw bytes since those are sent on the network thus + * avoiding possible information leaks of our PRNG. */ + uint8_t random_number[SR_RANDOM_NUMBER_LEN]; + /* Timestamp of commit. Correspond to TIMESTAMP. */ + uint64_t commit_ts; + /* This is the whole reveal message. We use it during verification */ + char encoded_reveal[SR_REVEAL_BASE64_LEN + 1]; +} sr_commit_t; + +/* API */ + +/* Public methods: */ + +int sr_init(int save_to_disk); +void sr_save_and_cleanup(void); +void sr_act_post_consensus(const networkstatus_t *consensus); +void sr_handle_received_commits(smartlist_t *commits, + crypto_pk_t *voter_key); +sr_commit_t *sr_parse_commit(const smartlist_t *args); +sr_srv_t *sr_parse_srv(const smartlist_t *args); +char *sr_get_string_for_vote(void); +char *sr_get_string_for_consensus(const smartlist_t *votes, + int32_t num_srv_agreements); +void sr_commit_free(sr_commit_t *commit); +void sr_srv_encode(char *dst, size_t dst_len, const sr_srv_t *srv); + +/* Private methods (only used by shared_random_state.c): */ +static inline +const char *sr_commit_get_rsa_fpr(const sr_commit_t *commit) +{ + return commit->rsa_identity_hex; +} + +void sr_compute_srv(void); +sr_commit_t *sr_generate_our_commit(time_t timestamp, + const authority_cert_t *my_rsa_cert); +#ifdef SHARED_RANDOM_PRIVATE + +/* Encode */ +STATIC int reveal_encode(const sr_commit_t *commit, char *dst, size_t len); +STATIC int commit_encode(const sr_commit_t *commit, char *dst, size_t len); +/* Decode. */ +STATIC int commit_decode(const char *encoded, sr_commit_t *commit); +STATIC int reveal_decode(const char *encoded, sr_commit_t *commit); + +STATIC int commit_has_reveal_value(const sr_commit_t *commit); + +STATIC int verify_commit_and_reveal(const sr_commit_t *commit); + +STATIC sr_srv_t *get_majority_srv_from_votes(const smartlist_t *votes, + int current); + +STATIC void save_commit_to_state(sr_commit_t *commit); +STATIC sr_srv_t *srv_dup(const sr_srv_t *orig); +STATIC int commitments_are_the_same(const sr_commit_t *commit_one, + const sr_commit_t *commit_two); +STATIC int commit_is_authoritative(const sr_commit_t *commit, + const char *voter_key); +STATIC int should_keep_commit(const sr_commit_t *commit, + const char *voter_key, + sr_phase_t phase); +STATIC void save_commit_during_reveal_phase(const sr_commit_t *commit); + +#endif /* SHARED_RANDOM_PRIVATE */ + +#ifdef TOR_UNIT_TESTS + +void set_num_srv_agreements(int32_t value); + +#endif /* TOR_UNIT_TESTS */ + +#endif /* TOR_SHARED_RANDOM_H */ + diff --git a/src/or/shared_random_state.c b/src/or/shared_random_state.c new file mode 100644 index 0000000000..52a0034db7 --- /dev/null +++ b/src/or/shared_random_state.c @@ -0,0 +1,1358 @@ +/* Copyright (c) 2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file shared_random_state.c + * + * \brief Functions and data structures for the state of the random protocol + * as defined in proposal #250. + **/ + +#define SHARED_RANDOM_STATE_PRIVATE + +#include "or.h" +#include "shared_random.h" +#include "config.h" +#include "confparse.h" +#include "dirvote.h" +#include "networkstatus.h" +#include "router.h" +#include "shared_random_state.h" + +/* Default filename of the shared random state on disk. */ +static const char default_fname[] = "sr-state"; + +/* String representation of a protocol phase. */ +static const char *phase_str[] = { "unknown", "commit", "reveal" }; + +/* Our shared random protocol state. There is only one possible state per + * protocol run so this is the global state which is reset at every run once + * the shared random value has been computed. */ +static sr_state_t *sr_state = NULL; + +/* Representation of our persistent state on disk. The sr_state above + * contains the data parsed from this state. When we save to disk, we + * translate the sr_state to this sr_disk_state. */ +static sr_disk_state_t *sr_disk_state = NULL; + +/* Disk state file keys. */ +static const char dstate_commit_key[] = "Commit"; +static const char dstate_prev_srv_key[] = "SharedRandPreviousValue"; +static const char dstate_cur_srv_key[] = "SharedRandCurrentValue"; + +/* These next two are duplicates or near-duplicates from config.c */ +#define VAR(name, conftype, member, initvalue) \ + { name, CONFIG_TYPE_ ## conftype, STRUCT_OFFSET(sr_disk_state_t, member), \ + initvalue } +/* As VAR, but the option name and member name are the same. */ +#define V(member, conftype, initvalue) \ + VAR(#member, conftype, member, initvalue) +/* Our persistent state magic number. */ +#define SR_DISK_STATE_MAGIC 0x98AB1254 +/* Each protocol phase has 12 rounds */ +#define SHARED_RANDOM_N_ROUNDS 12 +/* Number of phase we have in a protocol. */ +#define SHARED_RANDOM_N_PHASES 2 + +static int +disk_state_validate_cb(void *old_state, void *state, void *default_state, + int from_setconf, char **msg); + +/* Array of variables that are saved to disk as a persistent state. */ +static config_var_t state_vars[] = { + V(Version, UINT, "0"), + V(TorVersion, STRING, NULL), + V(ValidAfter, ISOTIME, NULL), + V(ValidUntil, ISOTIME, NULL), + + V(Commit, LINELIST, NULL), + + V(SharedRandValues, LINELIST_V, NULL), + VAR("SharedRandPreviousValue",LINELIST_S, SharedRandValues, NULL), + VAR("SharedRandCurrentValue", LINELIST_S, SharedRandValues, NULL), + { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL } +}; + +/* "Extra" variable in the state that receives lines we can't parse. This + * lets us preserve options from versions of Tor newer than us. */ +static config_var_t state_extra_var = { + "__extra", CONFIG_TYPE_LINELIST, + STRUCT_OFFSET(sr_disk_state_t, ExtraLines), NULL +}; + +/* Configuration format of sr_disk_state_t. */ +static const config_format_t state_format = { + sizeof(sr_disk_state_t), + SR_DISK_STATE_MAGIC, + STRUCT_OFFSET(sr_disk_state_t, magic_), + NULL, + state_vars, + disk_state_validate_cb, + &state_extra_var, +}; + +/* Return a string representation of a protocol phase. */ +STATIC const char * +get_phase_str(sr_phase_t phase) +{ + const char *the_string = NULL; + + switch (phase) { + case SR_PHASE_COMMIT: + case SR_PHASE_REVEAL: + the_string = phase_str[phase]; + break; + default: + /* Unknown phase shouldn't be possible. */ + tor_assert(0); + } + + return the_string; +} + +/* Return the voting interval of the tor vote subsystem. */ +static int +get_voting_interval(void) +{ + int interval; + networkstatus_t *consensus = networkstatus_get_live_consensus(time(NULL)); + + if (consensus) { + interval = (int)(consensus->fresh_until - consensus->valid_after); + } else { + /* Same for both a testing and real network. We voluntarily ignore the + * InitialVotingInterval since it complexifies things and it doesn't + * affect the SR protocol. */ + interval = get_options()->V3AuthVotingInterval; + } + tor_assert(interval > 0); + return interval; +} + +/* Given the time <b>now</b>, return the start time of the current round of + * the SR protocol. For example, if it's 23:47:08, the current round thus + * started at 23:47:00 for a voting interval of 10 seconds. */ +static time_t +get_start_time_of_current_round(time_t now) +{ + const or_options_t *options = get_options(); + int voting_interval = get_voting_interval(); + voting_schedule_t *new_voting_schedule = + get_voting_schedule(options, now, LOG_INFO); + tor_assert(new_voting_schedule); + + /* First, get the start time of the next round */ + time_t next_start = new_voting_schedule->interval_starts; + /* Now roll back next_start by a voting interval to find the start time of + the current round. */ + time_t curr_start = dirvote_get_start_of_next_interval( + next_start - voting_interval - 1, + voting_interval, + options->TestingV3AuthVotingStartOffset); + + tor_free(new_voting_schedule); + + return curr_start; +} + +/* Return the time we should expire the state file created at <b>now</b>. + * We expire the state file in the beginning of the next protocol run. */ +STATIC time_t +get_state_valid_until_time(time_t now) +{ + int total_rounds = SHARED_RANDOM_N_ROUNDS * SHARED_RANDOM_N_PHASES; + int current_round, voting_interval, rounds_left; + time_t valid_until, beginning_of_current_round; + + voting_interval = get_voting_interval(); + /* Find the time the current round started. */ + beginning_of_current_round = get_start_time_of_current_round(now); + + /* Find how many rounds are left till the end of the protocol run */ + current_round = (now / voting_interval) % total_rounds; + rounds_left = total_rounds - current_round; + + /* To find the valid-until time now, take the start time of the current + * round and add to it the time it takes for the leftover rounds to + * complete. */ + valid_until = beginning_of_current_round + (rounds_left * voting_interval); + + { /* Logging */ + char tbuf[ISO_TIME_LEN + 1]; + format_iso_time(tbuf, valid_until); + log_debug(LD_DIR, "SR: Valid until time for state set to %s.", tbuf); + } + + return valid_until; +} + +/* Given the consensus 'valid-after' time, return the protocol phase we should + * be in. */ +STATIC sr_phase_t +get_sr_protocol_phase(time_t valid_after) +{ + /* Shared random protocol has two phases, commit and reveal. */ + int total_periods = SHARED_RANDOM_N_ROUNDS * SHARED_RANDOM_N_PHASES; + int current_slot; + + /* Split time into slots of size 'voting_interval'. See which slot we are + * currently into, and find which phase it corresponds to. */ + current_slot = (valid_after / get_voting_interval()) % total_periods; + + if (current_slot < SHARED_RANDOM_N_ROUNDS) { + return SR_PHASE_COMMIT; + } else { + return SR_PHASE_REVEAL; + } +} + +/* Add the given <b>commit</b> to <b>state</b>. It MUST be a valid commit + * and there shouldn't be a commit from the same authority in the state + * already else verification hasn't been done prior. This takes ownership of + * the commit once in our state. */ +static void +commit_add_to_state(sr_commit_t *commit, sr_state_t *state) +{ + sr_commit_t *saved_commit; + + tor_assert(commit); + tor_assert(state); + + saved_commit = digestmap_set(state->commits, commit->rsa_identity, + commit); + if (saved_commit != NULL) { + /* This means we already have that commit in our state so adding twice + * the same commit is either a code flow error, a corrupted disk state + * or some new unknown issue. */ + log_warn(LD_DIR, "SR: Commit from %s exists in our state while " + "adding it: '%s'", sr_commit_get_rsa_fpr(commit), + commit->encoded_commit); + sr_commit_free(saved_commit); + } +} + +/* Helper: deallocate a commit object. (Used with digestmap_free(), which + * requires a function pointer whose argument is void *). */ +static void +commit_free_(void *p) +{ + sr_commit_free(p); +} + +/* Free a state that was allocated with state_new(). */ +static void +state_free(sr_state_t *state) +{ + if (state == NULL) { + return; + } + tor_free(state->fname); + digestmap_free(state->commits, commit_free_); + tor_free(state->current_srv); + tor_free(state->previous_srv); + tor_free(state); +} + +/* Allocate an sr_state_t object and returns it. If no <b>fname</b>, the + * default file name is used. This function does NOT initialize the state + * timestamp, phase or shared random value. NULL is never returned. */ +static sr_state_t * +state_new(const char *fname, time_t now) +{ + sr_state_t *new_state = tor_malloc_zero(sizeof(*new_state)); + /* If file name is not provided, use default. */ + if (fname == NULL) { + fname = default_fname; + } + new_state->fname = tor_strdup(fname); + new_state->version = SR_PROTO_VERSION; + new_state->commits = digestmap_new(); + new_state->phase = get_sr_protocol_phase(now); + new_state->valid_until = get_state_valid_until_time(now); + return new_state; +} + +/* Set our global state pointer with the one given. */ +static void +state_set(sr_state_t *state) +{ + tor_assert(state); + if (sr_state != NULL) { + state_free(sr_state); + } + sr_state = state; +} + +/* Free an allocated disk state. */ +static void +disk_state_free(sr_disk_state_t *state) +{ + if (state == NULL) { + return; + } + config_free(&state_format, state); +} + +/* Allocate a new disk state, initialize it and return it. */ +static sr_disk_state_t * +disk_state_new(time_t now) +{ + sr_disk_state_t *new_state = tor_malloc_zero(sizeof(*new_state)); + + new_state->magic_ = SR_DISK_STATE_MAGIC; + new_state->Version = SR_PROTO_VERSION; + new_state->TorVersion = tor_strdup(get_version()); + new_state->ValidUntil = get_state_valid_until_time(now); + new_state->ValidAfter = now; + + /* Init config format. */ + config_init(&state_format, new_state); + return new_state; +} + +/* Set our global disk state with the given state. */ +static void +disk_state_set(sr_disk_state_t *state) +{ + tor_assert(state); + if (sr_disk_state != NULL) { + disk_state_free(sr_disk_state); + } + sr_disk_state = state; +} + +/* Return -1 if the disk state is invalid (something in there that we can't or + * shouldn't use). Return 0 if everything checks out. */ +static int +disk_state_validate(const sr_disk_state_t *state) +{ + time_t now; + + tor_assert(state); + + /* Do we support the protocol version in the state or is it 0 meaning + * Version wasn't found in the state file or bad anyway ? */ + if (state->Version == 0 || state->Version > SR_PROTO_VERSION) { + goto invalid; + } + + /* If the valid until time is before now, we shouldn't use that state. */ + now = time(NULL); + if (state->ValidUntil < now) { + log_info(LD_DIR, "SR: Disk state has expired. Ignoring it."); + goto invalid; + } + + /* Make sure we don't have a valid after time that is earlier than a valid + * until time which would make things not work well. */ + if (state->ValidAfter >= state->ValidUntil) { + log_info(LD_DIR, "SR: Disk state valid after/until times are invalid."); + goto invalid; + } + + return 0; + + invalid: + return -1; +} + +/* Validate the disk state (NOP for now). */ +static int +disk_state_validate_cb(void *old_state, void *state, void *default_state, + int from_setconf, char **msg) +{ + /* We don't use these; only options do. */ + (void) from_setconf; + (void) default_state; + (void) old_state; + + /* This is called by config_dump which is just before we are about to + * write it to disk. At that point, our global memory state has been + * copied to the disk state so it's fair to assume it's trustable. */ + (void) state; + (void) msg; + return 0; +} + +/* Parse the Commit line(s) in the disk state and translate them to the + * the memory state. Return 0 on success else -1 on error. */ +static int +disk_state_parse_commits(sr_state_t *state, + const sr_disk_state_t *disk_state) +{ + config_line_t *line; + smartlist_t *args = NULL; + + tor_assert(state); + tor_assert(disk_state); + + for (line = disk_state->Commit; line; line = line->next) { + sr_commit_t *commit = NULL; + + /* Extra safety. */ + if (strcasecmp(line->key, dstate_commit_key) || + line->value == NULL) { + /* Ignore any lines that are not commits. */ + tor_fragile_assert(); + continue; + } + args = smartlist_new(); + smartlist_split_string(args, line->value, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + if (smartlist_len(args) < 3) { + log_warn(LD_BUG, "SR: Too few arguments in Commit Line: %s", + escaped(line->value)); + goto error; + } + commit = sr_parse_commit(args); + if (commit == NULL) { + /* Ignore badly formed commit. It could also be a authority + * fingerprint that we don't know about so it shouldn't be used. */ + continue; + } + /* We consider parseable commit from our disk state to be valid because + * they need to be in the first place to get in there. */ + commit->valid = 1; + /* Add commit to our state pointer. */ + commit_add_to_state(commit, state); + + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + } + + return 0; + + error: + SMARTLIST_FOREACH(args, char *, cp, tor_free(cp)); + smartlist_free(args); + return -1; +} + +/* Parse a share random value line from the disk state and save it to dst + * which is an allocated srv object. Return 0 on success else -1. */ +static int +disk_state_parse_srv(const char *value, sr_srv_t *dst) +{ + int ret = -1; + smartlist_t *args; + sr_srv_t *srv; + + tor_assert(value); + tor_assert(dst); + + args = smartlist_new(); + smartlist_split_string(args, value, " ", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + if (smartlist_len(args) < 2) { + log_warn(LD_BUG, "SR: Too few arguments in shared random value. " + "Line: %s", escaped(value)); + goto error; + } + srv = sr_parse_srv(args); + if (srv == NULL) { + goto error; + } + dst->num_reveals = srv->num_reveals; + memcpy(dst->value, srv->value, sizeof(dst->value)); + tor_free(srv); + ret = 0; + + error: + SMARTLIST_FOREACH(args, char *, s, tor_free(s)); + smartlist_free(args); + return ret; +} + +/* Parse both SharedRandCurrentValue and SharedRandPreviousValue line from + * the state. Return 0 on success else -1. */ +static int +disk_state_parse_sr_values(sr_state_t *state, + const sr_disk_state_t *disk_state) +{ + /* Only one value per type (current or previous) is allowed so we keep + * track of it with these flag. */ + unsigned int seen_previous = 0, seen_current = 0; + config_line_t *line; + sr_srv_t *srv = NULL; + + tor_assert(state); + tor_assert(disk_state); + + for (line = disk_state->SharedRandValues; line; line = line->next) { + if (line->value == NULL) { + continue; + } + srv = tor_malloc_zero(sizeof(*srv)); + if (disk_state_parse_srv(line->value, srv) < 0) { + log_warn(LD_BUG, "SR: Broken current SRV line in state %s", + escaped(line->value)); + goto bad; + } + if (!strcasecmp(line->key, dstate_prev_srv_key)) { + if (seen_previous) { + log_warn(LD_DIR, "SR: Second previous SRV value seen. Bad state"); + goto bad; + } + state->previous_srv = srv; + seen_previous = 1; + } else if (!strcasecmp(line->key, dstate_cur_srv_key)) { + if (seen_current) { + log_warn(LD_DIR, "SR: Second current SRV value seen. Bad state"); + goto bad; + } + state->current_srv = srv; + seen_current = 1; + } else { + /* Unknown key. Ignoring. */ + tor_free(srv); + } + } + + return 0; + bad: + tor_free(srv); + return -1; +} + +/* Parse the given disk state and set a newly allocated state. On success, + * return that state else NULL. */ +static sr_state_t * +disk_state_parse(const sr_disk_state_t *new_disk_state) +{ + sr_state_t *new_state = state_new(default_fname, time(NULL)); + + tor_assert(new_disk_state); + + new_state->version = new_disk_state->Version; + new_state->valid_until = new_disk_state->ValidUntil; + new_state->valid_after = new_disk_state->ValidAfter; + + /* Set our current phase according to the valid-after time in our disk + * state. The disk state we are parsing contains everything for the phase + * starting at valid_after so make sure our phase reflects that. */ + new_state->phase = get_sr_protocol_phase(new_state->valid_after); + + /* Parse the shared random values. */ + if (disk_state_parse_sr_values(new_state, new_disk_state) < 0) { + goto error; + } + /* Parse the commits. */ + if (disk_state_parse_commits(new_state, new_disk_state) < 0) { + goto error; + } + /* Great! This new state contains everything we had on disk. */ + return new_state; + + error: + state_free(new_state); + return NULL; +} + +/* From a valid commit object and an allocated config line, set the line's + * value to the state string representation of a commit. */ +static void +disk_state_put_commit_line(const sr_commit_t *commit, config_line_t *line) +{ + char *reveal_str = NULL; + + tor_assert(commit); + tor_assert(line); + + if (!tor_mem_is_zero(commit->encoded_reveal, + sizeof(commit->encoded_reveal))) { + /* Add extra whitespace so we can format the line correctly. */ + tor_asprintf(&reveal_str, " %s", commit->encoded_reveal); + } + tor_asprintf(&line->value, "%u %s %s %s%s", + SR_PROTO_VERSION, + crypto_digest_algorithm_get_name(commit->alg), + sr_commit_get_rsa_fpr(commit), + commit->encoded_commit, + reveal_str != NULL ? reveal_str : ""); + if (reveal_str != NULL) { + memwipe(reveal_str, 0, strlen(reveal_str)); + tor_free(reveal_str); + } +} + +/* From a valid srv object and an allocated config line, set the line's + * value to the state string representation of a shared random value. */ +static void +disk_state_put_srv_line(const sr_srv_t *srv, config_line_t *line) +{ + char encoded[SR_SRV_VALUE_BASE64_LEN + 1]; + + tor_assert(line); + + /* No SRV value thus don't add the line. This is possible since we might + * not have a current or previous SRV value in our state. */ + if (srv == NULL) { + return; + } + sr_srv_encode(encoded, sizeof(encoded), srv); + tor_asprintf(&line->value, "%" PRIu64 " %s", srv->num_reveals, encoded); +} + +/* Reset disk state that is free allocated memory and zeroed the object. */ +static void +disk_state_reset(void) +{ + /* Free allocated memory */ + config_free_lines(sr_disk_state->Commit); + config_free_lines(sr_disk_state->SharedRandValues); + config_free_lines(sr_disk_state->ExtraLines); + tor_free(sr_disk_state->TorVersion); + + /* Clean up the struct */ + memset(sr_disk_state, 0, sizeof(*sr_disk_state)); + + /* Reset it with useful data */ + sr_disk_state->magic_ = SR_DISK_STATE_MAGIC; + sr_disk_state->TorVersion = tor_strdup(get_version()); +} + +/* Update our disk state based on our global SR state. */ +static void +disk_state_update(void) +{ + config_line_t **next, *line; + + tor_assert(sr_disk_state); + tor_assert(sr_state); + + /* Reset current disk state. */ + disk_state_reset(); + + /* First, update elements that we don't need to do a construction. */ + sr_disk_state->Version = sr_state->version; + sr_disk_state->ValidUntil = sr_state->valid_until; + sr_disk_state->ValidAfter = sr_state->valid_after; + + /* Shared random values. */ + next = &sr_disk_state->SharedRandValues; + if (sr_state->previous_srv != NULL) { + *next = line = tor_malloc_zero(sizeof(config_line_t)); + line->key = tor_strdup(dstate_prev_srv_key); + disk_state_put_srv_line(sr_state->previous_srv, line); + /* Go to the next shared random value. */ + next = &(line->next); + } + if (sr_state->current_srv != NULL) { + *next = line = tor_malloc_zero(sizeof(*line)); + line->key = tor_strdup(dstate_cur_srv_key); + disk_state_put_srv_line(sr_state->current_srv, line); + } + + /* Parse the commits and construct config line(s). */ + next = &sr_disk_state->Commit; + DIGESTMAP_FOREACH(sr_state->commits, key, sr_commit_t *, commit) { + *next = line = tor_malloc_zero(sizeof(*line)); + line->key = tor_strdup(dstate_commit_key); + disk_state_put_commit_line(commit, line); + next = &(line->next); + } DIGESTMAP_FOREACH_END; +} + +/* Load state from disk and put it into our disk state. If the state passes + * validation, our global state will be updated with it. Return 0 on + * success. On error, -EINVAL is returned if the state on disk did contained + * something malformed or is unreadable. -ENOENT is returned indicating that + * the state file is either empty of non existing. */ +static int +disk_state_load_from_disk(void) +{ + int ret; + char *fname; + + fname = get_datadir_fname(default_fname); + ret = disk_state_load_from_disk_impl(fname); + tor_free(fname); + + return ret; +} + +/* Helper for disk_state_load_from_disk(). */ +STATIC int +disk_state_load_from_disk_impl(const char *fname) +{ + int ret; + char *content = NULL; + sr_state_t *parsed_state = NULL; + sr_disk_state_t *disk_state = NULL; + + /* Read content of file so we can parse it. */ + if ((content = read_file_to_str(fname, 0, NULL)) == NULL) { + log_warn(LD_FS, "SR: Unable to read SR state file %s", + escaped(fname)); + ret = -errno; + goto error; + } + + { + config_line_t *lines = NULL; + char *errmsg = NULL; + + /* Every error in this code path will return EINVAL. */ + ret = -EINVAL; + if (config_get_lines(content, &lines, 0) < 0) { + config_free_lines(lines); + goto error; + } + + disk_state = disk_state_new(time(NULL)); + config_assign(&state_format, disk_state, lines, 0, 0, &errmsg); + config_free_lines(lines); + if (errmsg) { + log_warn(LD_DIR, "SR: Reading state error: %s", errmsg); + tor_free(errmsg); + goto error; + } + } + + /* So far so good, we've loaded our state file into our disk state. Let's + * validate it and then parse it. */ + if (disk_state_validate(disk_state) < 0) { + ret = -EINVAL; + goto error; + } + + parsed_state = disk_state_parse(disk_state); + if (parsed_state == NULL) { + ret = -EINVAL; + goto error; + } + state_set(parsed_state); + disk_state_set(disk_state); + tor_free(content); + log_info(LD_DIR, "SR: State loaded successfully from file %s", fname); + return 0; + + error: + disk_state_free(disk_state); + tor_free(content); + return ret; +} + +/* Save the disk state to disk but before that update it from the current + * state so we always have the latest. Return 0 on success else -1. */ +static int +disk_state_save_to_disk(void) +{ + int ret; + char *state, *content = NULL, *fname = NULL; + char tbuf[ISO_TIME_LEN + 1]; + time_t now = time(NULL); + + /* If we didn't have the opportunity to setup an internal disk state, + * don't bother saving something to disk. */ + if (sr_disk_state == NULL) { + ret = 0; + goto done; + } + + /* Make sure that our disk state is up to date with our memory state + * before saving it to disk. */ + disk_state_update(); + state = config_dump(&state_format, NULL, sr_disk_state, 0, 0); + format_local_iso_time(tbuf, now); + tor_asprintf(&content, + "# Tor shared random state file last generated on %s " + "local time\n" + "# Other times below are in UTC\n" + "# Please *do not* edit this file.\n\n%s", + tbuf, state); + tor_free(state); + fname = get_datadir_fname(default_fname); + if (write_str_to_file(fname, content, 0) < 0) { + log_warn(LD_FS, "SR: Unable to write SR state to file %s", fname); + ret = -1; + goto done; + } + ret = 0; + log_debug(LD_DIR, "SR: Saved state to file %s", fname); + + done: + tor_free(fname); + tor_free(content); + return ret; +} + +/* Reset our state to prepare for a new protocol run. Once this returns, all + * commits in the state will be removed and freed. */ +STATIC void +reset_state_for_new_protocol_run(time_t valid_after) +{ + tor_assert(sr_state); + + /* Keep counters in track */ + sr_state->n_reveal_rounds = 0; + sr_state->n_commit_rounds = 0; + sr_state->n_protocol_runs++; + + /* Reset valid-until */ + sr_state->valid_until = get_state_valid_until_time(valid_after); + sr_state->valid_after = valid_after; + + /* We are in a new protocol run so cleanup commits. */ + sr_state_delete_commits(); +} + +/* This is the first round of the new protocol run starting at + * <b>valid_after</b>. Do the necessary housekeeping. */ +STATIC void +new_protocol_run(time_t valid_after) +{ + sr_commit_t *our_commitment = NULL; + + /* Only compute the srv at the end of the reveal phase. */ + if (sr_state->phase == SR_PHASE_REVEAL) { + /* We are about to compute a new shared random value that will be set in + * our state as the current value so rotate values. */ + state_rotate_srv(); + /* Compute the shared randomness value of the day. */ + sr_compute_srv(); + } + + /* Prepare for the new protocol run by reseting the state */ + reset_state_for_new_protocol_run(valid_after); + + /* Do some logging */ + log_info(LD_DIR, "SR: Protocol run #%" PRIu64 " starting!", + sr_state->n_protocol_runs); + + /* Generate fresh commitments for this protocol run */ + our_commitment = sr_generate_our_commit(valid_after, + get_my_v3_authority_cert()); + if (our_commitment) { + /* Add our commitment to our state. In case we are unable to create one + * (highly unlikely), we won't vote for this protocol run since our + * commitment won't be in our state. */ + sr_state_add_commit(our_commitment); + } +} + +/* Return 1 iff the <b>next_phase</b> is a phase transition from the current + * phase that is it's different. */ +STATIC int +is_phase_transition(sr_phase_t next_phase) +{ + return sr_state->phase != next_phase; +} + +/* Helper function: return a commit using the RSA fingerprint of the + * authority or NULL if no such commit is known. */ +static sr_commit_t * +state_query_get_commit(const char *rsa_fpr) +{ + tor_assert(rsa_fpr); + return digestmap_get(sr_state->commits, rsa_fpr); +} + +/* Helper function: This handles the GET state action using an + * <b>obj_type</b> and <b>data</b> needed for the action. */ +static void * +state_query_get_(sr_state_object_t obj_type, const void *data) +{ + void *obj = NULL; + + switch (obj_type) { + case SR_STATE_OBJ_COMMIT: + { + obj = state_query_get_commit(data); + break; + } + case SR_STATE_OBJ_COMMITS: + obj = sr_state->commits; + break; + case SR_STATE_OBJ_CURSRV: + obj = sr_state->current_srv; + break; + case SR_STATE_OBJ_PREVSRV: + obj = sr_state->previous_srv; + break; + case SR_STATE_OBJ_PHASE: + obj = &sr_state->phase; + break; + case SR_STATE_OBJ_VALID_AFTER: + default: + tor_assert(0); + } + return obj; +} + +/* Helper function: This handles the PUT state action using an + * <b>obj_type</b> and <b>data</b> needed for the action. */ +static void +state_query_put_(sr_state_object_t obj_type, void *data) +{ + switch (obj_type) { + case SR_STATE_OBJ_COMMIT: + { + sr_commit_t *commit = data; + tor_assert(commit); + commit_add_to_state(commit, sr_state); + break; + } + case SR_STATE_OBJ_CURSRV: + sr_state->current_srv = (sr_srv_t *) data; + break; + case SR_STATE_OBJ_PREVSRV: + sr_state->previous_srv = (sr_srv_t *) data; + break; + case SR_STATE_OBJ_VALID_AFTER: + sr_state->valid_after = *((time_t *) data); + break; + /* It's not allowed to change the phase nor the full commitments map from + * the state. The phase is decided during a strict process post voting and + * the commits should be put individually. */ + case SR_STATE_OBJ_PHASE: + case SR_STATE_OBJ_COMMITS: + default: + tor_assert(0); + } +} + +/* Helper function: This handles the DEL_ALL state action using an + * <b>obj_type</b> and <b>data</b> needed for the action. */ +static void +state_query_del_all_(sr_state_object_t obj_type) +{ + switch (obj_type) { + case SR_STATE_OBJ_COMMIT: + { + /* We are in a new protocol run so cleanup commitments. */ + DIGESTMAP_FOREACH_MODIFY(sr_state->commits, key, sr_commit_t *, c) { + sr_commit_free(c); + MAP_DEL_CURRENT(key); + } DIGESTMAP_FOREACH_END; + break; + } + /* The following object are _NOT_ suppose to be removed. */ + case SR_STATE_OBJ_CURSRV: + case SR_STATE_OBJ_PREVSRV: + case SR_STATE_OBJ_PHASE: + case SR_STATE_OBJ_COMMITS: + case SR_STATE_OBJ_VALID_AFTER: + default: + tor_assert(0); + } +} + +/* Helper function: This handles the DEL state action using an + * <b>obj_type</b> and <b>data</b> needed for the action. */ +static void +state_query_del_(sr_state_object_t obj_type, void *data) +{ + (void) data; + + switch (obj_type) { + case SR_STATE_OBJ_PREVSRV: + tor_free(sr_state->previous_srv); + break; + case SR_STATE_OBJ_CURSRV: + tor_free(sr_state->current_srv); + break; + case SR_STATE_OBJ_COMMIT: + case SR_STATE_OBJ_COMMITS: + case SR_STATE_OBJ_PHASE: + case SR_STATE_OBJ_VALID_AFTER: + default: + tor_assert(0); + } +} + +/* Query state using an <b>action</b> for an object type <b>obj_type</b>. + * The <b>data</b> pointer needs to point to an object that the action needs + * to use and if anything is required to be returned, it is stored in + * <b>out</b>. + * + * This mechanism exists so we have one single point where we synchronized + * our memory state with our disk state for every actions that changes it. + * We then trigger a write on disk immediately. + * + * This should be the only entry point to our memory state. It's used by all + * our state accessors and should be in the future. */ +static void +state_query(sr_state_action_t action, sr_state_object_t obj_type, + void *data, void **out) +{ + switch (action) { + case SR_STATE_ACTION_GET: + *out = state_query_get_(obj_type, data); + break; + case SR_STATE_ACTION_PUT: + state_query_put_(obj_type, data); + break; + case SR_STATE_ACTION_DEL: + state_query_del_(obj_type, data); + break; + case SR_STATE_ACTION_DEL_ALL: + state_query_del_all_(obj_type); + break; + case SR_STATE_ACTION_SAVE: + /* Only trigger a disk state save. */ + break; + default: + tor_assert(0); + } + + /* If the action actually changes the state, immediately save it to disk. + * The following will sync the state -> disk state and then save it. */ + if (action != SR_STATE_ACTION_GET) { + disk_state_save_to_disk(); + } +} + +/* Delete the current SRV value from the state freeing it and the value is set + * to NULL meaning empty. */ +static void +state_del_current_srv(void) +{ + state_query(SR_STATE_ACTION_DEL, SR_STATE_OBJ_CURSRV, NULL, NULL); +} + +/* Delete the previous SRV value from the state freeing it and the value is + * set to NULL meaning empty. */ +static void +state_del_previous_srv(void) +{ + state_query(SR_STATE_ACTION_DEL, SR_STATE_OBJ_PREVSRV, NULL, NULL); +} + +/* Rotate SRV value by freeing the previous value, assigning the current + * value to the previous one and nullifying the current one. */ +STATIC void +state_rotate_srv(void) +{ + /* First delete previous SRV from the state. Object will be freed. */ + state_del_previous_srv(); + /* Set previous SRV with the current one. */ + sr_state_set_previous_srv(sr_state_get_current_srv()); + /* Nullify the current srv. */ + sr_state_set_current_srv(NULL); +} + +/* Set valid after time in the our state. */ +void +sr_state_set_valid_after(time_t valid_after) +{ + state_query(SR_STATE_ACTION_PUT, SR_STATE_OBJ_VALID_AFTER, + (void *) &valid_after, NULL); +} + +/* Return the phase we are currently in according to our state. */ +sr_phase_t +sr_state_get_phase(void) +{ + void *ptr; + state_query(SR_STATE_ACTION_GET, SR_STATE_OBJ_PHASE, NULL, &ptr); + return *(sr_phase_t *) ptr; +} + +/* Return the previous SRV value from our state. Value CAN be NULL. */ +const sr_srv_t * +sr_state_get_previous_srv(void) +{ + const sr_srv_t *srv; + state_query(SR_STATE_ACTION_GET, SR_STATE_OBJ_PREVSRV, NULL, + (void *) &srv); + return srv; +} + +/* Set the current SRV value from our state. Value CAN be NULL. The srv + * object ownership is transfered to the state object. */ +void +sr_state_set_previous_srv(const sr_srv_t *srv) +{ + state_query(SR_STATE_ACTION_PUT, SR_STATE_OBJ_PREVSRV, (void *) srv, + NULL); +} + +/* Return the current SRV value from our state. Value CAN be NULL. */ +const sr_srv_t * +sr_state_get_current_srv(void) +{ + const sr_srv_t *srv; + state_query(SR_STATE_ACTION_GET, SR_STATE_OBJ_CURSRV, NULL, + (void *) &srv); + return srv; +} + +/* Set the current SRV value from our state. Value CAN be NULL. The srv + * object ownership is transfered to the state object. */ +void +sr_state_set_current_srv(const sr_srv_t *srv) +{ + state_query(SR_STATE_ACTION_PUT, SR_STATE_OBJ_CURSRV, (void *) srv, + NULL); +} + +/* Clean all the SRVs in our state. */ +void +sr_state_clean_srvs(void) +{ + /* Remove SRVs from state. They will be set to NULL as "empty". */ + state_del_previous_srv(); + state_del_current_srv(); +} + +/* Return a pointer to the commits map from our state. CANNOT be NULL. */ +digestmap_t * +sr_state_get_commits(void) +{ + digestmap_t *commits; + state_query(SR_STATE_ACTION_GET, SR_STATE_OBJ_COMMITS, + NULL, (void *) &commits); + tor_assert(commits); + return commits; +} + +/* Update the current SR state as needed for the upcoming voting round at + * <b>valid_after</b>. */ +void +sr_state_update(time_t valid_after) +{ + sr_phase_t next_phase; + + tor_assert(sr_state); + + /* Don't call this function twice in the same voting period. */ + if (valid_after <= sr_state->valid_after) { + log_info(LD_DIR, "SR: Asked to update state twice. Ignoring."); + return; + } + + /* Get phase of upcoming round. */ + next_phase = get_sr_protocol_phase(valid_after); + + /* If we are transitioning to a new protocol phase, prepare the stage. */ + if (is_phase_transition(next_phase)) { + if (next_phase == SR_PHASE_COMMIT) { + /* Going into commit phase means we are starting a new protocol run. */ + new_protocol_run(valid_after); + } + /* Set the new phase for this round */ + sr_state->phase = next_phase; + } else if (sr_state->phase == SR_PHASE_COMMIT && + digestmap_size(sr_state->commits) == 0) { + /* We are _NOT_ in a transition phase so if we are in the commit phase + * and have no commit, generate one. Chances are that we are booting up + * so let's have a commit in our state for the next voting period. */ + sr_commit_t *our_commit = + sr_generate_our_commit(valid_after, get_my_v3_authority_cert()); + if (our_commit) { + /* Add our commitment to our state. In case we are unable to create one + * (highly unlikely), we won't vote for this protocol run since our + * commitment won't be in our state. */ + sr_state_add_commit(our_commit); + } + } + + sr_state_set_valid_after(valid_after); + + /* Count the current round */ + if (sr_state->phase == SR_PHASE_COMMIT) { + /* invariant check: we've not entered reveal phase yet */ + tor_assert(sr_state->n_reveal_rounds == 0); + sr_state->n_commit_rounds++; + } else { + sr_state->n_reveal_rounds++; + } + + { /* Debugging. */ + char tbuf[ISO_TIME_LEN + 1]; + format_iso_time(tbuf, valid_after); + log_info(LD_DIR, "SR: State prepared for upcoming voting period (%s). " + "Upcoming phase is %s (counters: %d commit & %d reveal rounds).", + tbuf, get_phase_str(sr_state->phase), + sr_state->n_commit_rounds, sr_state->n_reveal_rounds); + } +} + +/* Return commit object from the given authority digest <b>rsa_identity</b>. + * Return NULL if not found. */ +sr_commit_t * +sr_state_get_commit(const char *rsa_identity) +{ + sr_commit_t *commit; + + tor_assert(rsa_identity); + + state_query(SR_STATE_ACTION_GET, SR_STATE_OBJ_COMMIT, + (void *) rsa_identity, (void *) &commit); + return commit; +} + +/* Add <b>commit</b> to the permanent state. The commit object ownership is + * transfered to the state so the caller MUST not free it. */ +void +sr_state_add_commit(sr_commit_t *commit) +{ + tor_assert(commit); + + /* Put the commit to the global state. */ + state_query(SR_STATE_ACTION_PUT, SR_STATE_OBJ_COMMIT, + (void *) commit, NULL); + + log_debug(LD_DIR, "SR: Commit from %s has been added to our state.", + sr_commit_get_rsa_fpr(commit)); +} + +/* Remove all commits from our state. */ +void +sr_state_delete_commits(void) +{ + state_query(SR_STATE_ACTION_DEL_ALL, SR_STATE_OBJ_COMMIT, NULL, NULL); +} + +/* Copy the reveal information from <b>commit</b> into <b>saved_commit</b>. + * This <b>saved_commit</b> MUST come from our current SR state. Once modified, + * the disk state is updated. */ +void +sr_state_copy_reveal_info(sr_commit_t *saved_commit, const sr_commit_t *commit) +{ + tor_assert(saved_commit); + tor_assert(commit); + + saved_commit->reveal_ts = commit->reveal_ts; + memcpy(saved_commit->random_number, commit->random_number, + sizeof(saved_commit->random_number)); + + strlcpy(saved_commit->encoded_reveal, commit->encoded_reveal, + sizeof(saved_commit->encoded_reveal)); + state_query(SR_STATE_ACTION_SAVE, 0, NULL, NULL); + log_debug(LD_DIR, "SR: Reveal value learned %s (for commit %s) from %s", + saved_commit->encoded_reveal, saved_commit->encoded_commit, + sr_commit_get_rsa_fpr(saved_commit)); +} + +/* Set the fresh SRV flag from our state. This doesn't need to trigger a + * disk state synchronization so we directly change the state. */ +void +sr_state_set_fresh_srv(void) +{ + sr_state->is_srv_fresh = 1; +} + +/* Unset the fresh SRV flag from our state. This doesn't need to trigger a + * disk state synchronization so we directly change the state. */ +void +sr_state_unset_fresh_srv(void) +{ + sr_state->is_srv_fresh = 0; +} + +/* Return the value of the fresh SRV flag. */ +unsigned int +sr_state_srv_is_fresh(void) +{ + return sr_state->is_srv_fresh; +} + +/* Cleanup and free our disk and memory state. */ +void +sr_state_free(void) +{ + state_free(sr_state); + disk_state_free(sr_disk_state); + /* Nullify our global state. */ + sr_state = NULL; + sr_disk_state = NULL; +} + +/* Save our current state in memory to disk. */ +void +sr_state_save(void) +{ + /* Query a SAVE action on our current state so it's synced and saved. */ + state_query(SR_STATE_ACTION_SAVE, 0, NULL, NULL); +} + +/* Return 1 iff the state has been initialized that is it exists in memory. + * Return 0 otherwise. */ +int +sr_state_is_initialized(void) +{ + return sr_state == NULL ? 0 : 1; +} + +/* Initialize the disk and memory state. + * + * If save_to_disk is set to 1, the state is immediately saved to disk after + * creation else it's not thus only kept in memory. + * If read_from_disk is set to 1, we try to load the state from the disk and + * if not found, a new state is created. + * + * Return 0 on success else a negative value on error. */ +int +sr_state_init(int save_to_disk, int read_from_disk) +{ + int ret = -ENOENT; + time_t now = time(NULL); + + /* We shouldn't have those assigned. */ + tor_assert(sr_disk_state == NULL); + tor_assert(sr_state == NULL); + + /* First, try to load the state from disk. */ + if (read_from_disk) { + ret = disk_state_load_from_disk(); + } + + if (ret < 0) { + switch (-ret) { + case EINVAL: + /* We have a state on disk but it contains something we couldn't parse + * or an invalid entry in the state file. Let's remove it since it's + * obviously unusable and replace it by an new fresh state below. */ + case ENOENT: + { + /* No state on disk so allocate our states for the first time. */ + sr_state_t *new_state = state_new(default_fname, now); + sr_disk_state_t *new_disk_state = disk_state_new(now); + state_set(new_state); + /* It's important to set our disk state pointer since the save call + * below uses it to synchronized it with our memory state. */ + disk_state_set(new_disk_state); + /* No entry, let's save our new state to disk. */ + if (save_to_disk && disk_state_save_to_disk() < 0) { + goto error; + } + break; + } + default: + /* Big problem. Not possible. */ + tor_assert(0); + } + } + /* We have a state in memory, let's make sure it's updated for the current + * and next voting round. */ + { + time_t valid_after = get_next_valid_after_time(now); + sr_state_update(valid_after); + } + return 0; + + error: + return -1; +} + +#ifdef TOR_UNIT_TESTS + +/* Set the current phase of the protocol. Used only by unit tests. */ +void +set_sr_phase(sr_phase_t phase) +{ + tor_assert(sr_state); + sr_state->phase = phase; +} + +/* Get the SR state. Used only by unit tests */ +sr_state_t * +get_sr_state(void) +{ + return sr_state; +} + +#endif /* TOR_UNIT_TESTS */ + diff --git a/src/or/shared_random_state.h b/src/or/shared_random_state.h new file mode 100644 index 0000000000..43a7f1d284 --- /dev/null +++ b/src/or/shared_random_state.h @@ -0,0 +1,147 @@ +/* Copyright (c) 2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#ifndef TOR_SHARED_RANDOM_STATE_H +#define TOR_SHARED_RANDOM_STATE_H + +#include "shared_random.h" + +/* Action that can be performed on the state for any objects. */ +typedef enum { + SR_STATE_ACTION_GET = 1, + SR_STATE_ACTION_PUT = 2, + SR_STATE_ACTION_DEL = 3, + SR_STATE_ACTION_DEL_ALL = 4, + SR_STATE_ACTION_SAVE = 5, +} sr_state_action_t; + +/* Object in the state that can be queried through the state API. */ +typedef enum { + /* Will return a single commit using an authority identity key. */ + SR_STATE_OBJ_COMMIT, + /* Returns the entire list of commits from the state. */ + SR_STATE_OBJ_COMMITS, + /* Return the current SRV object pointer. */ + SR_STATE_OBJ_CURSRV, + /* Return the previous SRV object pointer. */ + SR_STATE_OBJ_PREVSRV, + /* Return the phase. */ + SR_STATE_OBJ_PHASE, + /* Get or Put the valid after time. */ + SR_STATE_OBJ_VALID_AFTER, +} sr_state_object_t; + +/* State of the protocol. It's also saved on disk in fname. This data + * structure MUST be synchronized at all time with the one on disk. */ +typedef struct sr_state_t { + /* Filename of the state file on disk. */ + char *fname; + /* Version of the protocol. */ + uint32_t version; + /* The valid-after of the voting period we have prepared the state for. */ + time_t valid_after; + /* Until when is this state valid? */ + time_t valid_until; + /* Protocol phase. */ + sr_phase_t phase; + + /* Number of runs completed. */ + uint64_t n_protocol_runs; + /* The number of commitment rounds we've performed in this protocol run. */ + unsigned int n_commit_rounds; + /* The number of reveal rounds we've performed in this protocol run. */ + unsigned int n_reveal_rounds; + + /* A map of all the received commitments for this protocol run. This is + * indexed by authority RSA identity digest. */ + digestmap_t *commits; + + /* Current and previous shared random value. */ + sr_srv_t *previous_srv; + sr_srv_t *current_srv; + + /* Indicate if the state contains an SRV that was _just_ generated. This is + * used during voting so that we know whether to use the super majority rule + * or not when deciding on keeping it for the consensus. It is _always_ set + * to 0 post consensus. + * + * EDGE CASE: if an authority computes a new SRV then immediately reboots + * and, once back up, votes for the current round, it won't know if the + * SRV is fresh or not ultimately making it _NOT_ use the super majority + * when deciding to put or not the SRV in the consensus. This is for now + * an acceptable very rare edge case. */ + unsigned int is_srv_fresh:1; +} sr_state_t; + +/* Persistent state of the protocol, as saved to disk. */ +typedef struct sr_disk_state_t { + uint32_t magic_; + /* Version of the protocol. */ + uint32_t Version; + /* Version of our running tor. */ + char *TorVersion; + /* Creation time of this state */ + time_t ValidAfter; + /* State valid until? */ + time_t ValidUntil; + /* All commits seen that are valid. */ + config_line_t *Commit; + /* Previous and current shared random value. */ + config_line_t *SharedRandValues; + /* Extra Lines for configuration we might not know. */ + config_line_t *ExtraLines; +} sr_disk_state_t; + +/* API */ + +/* Public methods: */ + +void sr_state_update(time_t valid_after); + +/* Private methods (only used by shared-random.c): */ + +void sr_state_set_valid_after(time_t valid_after); +sr_phase_t sr_state_get_phase(void); +const sr_srv_t *sr_state_get_previous_srv(void); +const sr_srv_t *sr_state_get_current_srv(void); +void sr_state_set_previous_srv(const sr_srv_t *srv); +void sr_state_set_current_srv(const sr_srv_t *srv); +void sr_state_clean_srvs(void); +digestmap_t *sr_state_get_commits(void); +sr_commit_t *sr_state_get_commit(const char *rsa_fpr); +void sr_state_add_commit(sr_commit_t *commit); +void sr_state_delete_commits(void); +void sr_state_copy_reveal_info(sr_commit_t *saved_commit, + const sr_commit_t *commit); +unsigned int sr_state_srv_is_fresh(void); +void sr_state_set_fresh_srv(void); +void sr_state_unset_fresh_srv(void); +int sr_state_init(int save_to_disk, int read_from_disk); +int sr_state_is_initialized(void); +void sr_state_save(void); +void sr_state_free(void); + +#ifdef SHARED_RANDOM_STATE_PRIVATE + +STATIC int disk_state_load_from_disk_impl(const char *fname); + +STATIC sr_phase_t get_sr_protocol_phase(time_t valid_after); + +STATIC time_t get_state_valid_until_time(time_t now); +STATIC const char *get_phase_str(sr_phase_t phase); +STATIC void reset_state_for_new_protocol_run(time_t valid_after); +STATIC void new_protocol_run(time_t valid_after); +STATIC void state_rotate_srv(void); +STATIC int is_phase_transition(sr_phase_t next_phase); + +#endif /* SHARED_RANDOM_STATE_PRIVATE */ + +#ifdef TOR_UNIT_TESTS + +STATIC void set_sr_phase(sr_phase_t phase); +STATIC sr_state_t *get_sr_state(void); + +#endif /* TOR_UNIT_TESTS */ + +#endif /* TOR_SHARED_RANDOM_STATE_H */ + diff --git a/src/or/tor_main.c b/src/or/tor_main.c index ac32eef559..21fbe3efb5 100644 --- a/src/or/tor_main.c +++ b/src/or/tor_main.c @@ -3,6 +3,8 @@ * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +extern const char tor_git_revision[]; + /** String describing which Tor Git repository version the source was * built from. This string is generated by a bit of shell kludging in * src/or/include.am, and is usually right. diff --git a/src/or/transports.c b/src/or/transports.c index 1b8b1e678c..c99ff1729b 100644 --- a/src/or/transports.c +++ b/src/or/transports.c @@ -1270,7 +1270,7 @@ get_transport_options_for_server_proxy(const managed_proxy_t *mp) /** Return the string that tor should place in TOR_PT_SERVER_BINDADDR * while configuring the server managed proxy in <b>mp</b>. The - * string is stored in the heap, and it's the the responsibility of + * string is stored in the heap, and it's the responsibility of * the caller to deallocate it after its use. */ static char * get_bindaddr_for_server_proxy(const managed_proxy_t *mp) @@ -1363,7 +1363,7 @@ create_managed_proxy_environment(const managed_proxy_t *mp) } } - /* XXX024 Remove the '=' here once versions of obfsproxy which + /* XXXX Remove the '=' here once versions of obfsproxy which * assert that this env var exists are sufficiently dead. * * (If we remove this line entirely, some joker will stick this diff --git a/src/test/bench.c b/src/test/bench.c index 5aefda5ff2..5595988f31 100644 --- a/src/test/bench.c +++ b/src/test/bench.c @@ -3,6 +3,7 @@ * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +extern const char tor_git_revision[]; /* Ordinarily defined in tor_main.c; this bit is just here to provide one * since we're not linking to tor_main.c */ const char tor_git_revision[] = ""; diff --git a/src/test/example_extrainfo.inc b/src/test/example_extrainfo.inc index e096afd6c4..0bf2341ef5 100644 --- a/src/test/example_extrainfo.inc +++ b/src/test/example_extrainfo.inc @@ -133,7 +133,7 @@ static const char EX_EI_BAD_NICKNAME_KEY[] = "/UBWNSyXCFDMqnddb/LZ8+VgttmxfYkpeRzSSmDijN3RbOvYJhhBAgMBAAE=\n" "-----END RSA PUBLIC KEY-----\n"; -const char EX_EI_BAD_TOKENS[] = +static const char EX_EI_BAD_TOKENS[] = "extra-info bob 6F314FB01A31162BD5E473D4977AC570DC5B86BB\n" "published 2014-10-05 20:07:00\n" "published 2014-10-05 20:07:00\n" @@ -145,8 +145,9 @@ const char EX_EI_BAD_TOKENS[] = "-----END SIGNATURE-----\n" ; -const char EX_EI_BAD_TOKENS_FP[] = "6F314FB01A31162BD5E473D4977AC570DC5B86BB"; -const char EX_EI_BAD_TOKENS_KEY[] = +static const char EX_EI_BAD_TOKENS_FP[] = + "6F314FB01A31162BD5E473D4977AC570DC5B86BB"; +static const char EX_EI_BAD_TOKENS_KEY[] = "-----BEGIN RSA PUBLIC KEY-----\n" "MIGJAoGBAL7Z8tz45Tb4tnEFS2sAyjubBV/giSfZdmXRkDV8Jo4xqWqhWFJn7+zN\n" "AXBWBThGeVH2WXrpz5seNJXgZJPxMTMsrnSCGcRXZw0Npti2MkLuQ6+prZa+OPwE\n" @@ -210,7 +211,8 @@ static const char EX_EI_GOOD_ED_EI[] = "\n" "\n" ; -const char EX_EI_GOOD_ED_EI_FP[] = "A692FE045C32B5E3A54B52882EF678A9DAC46A73"; +static const char EX_EI_GOOD_ED_EI_FP[] = + "A692FE045C32B5E3A54B52882EF678A9DAC46A73"; static const char EX_EI_GOOD_ED_EI_KEY[] = "-----BEGIN RSA PUBLIC KEY-----\n" "MIGJAoGBAM3jdYwjwGxDWYj/vyFkQT7RgeCNIn89Ei6D2+L/fdtFnqrMXOreFFHL\n" @@ -237,7 +239,8 @@ static const char EX_EI_ED_MISSING_SIG[] = "\n" "\n" ; -const char EX_EI_ED_MISSING_SIG_FP[] = "2A7521497B91A8437021515308A47491164EDBA1"; +static const char EX_EI_ED_MISSING_SIG_FP[] = + "2A7521497B91A8437021515308A47491164EDBA1"; static const char EX_EI_ED_MISSING_SIG_KEY[] = "-----BEGIN RSA PUBLIC KEY-----\n" "MIGJAoGBAOOB8ccxbtk2dB5FuKFhGndDcO6STNjB6KiG0b9X2QwKrOZMfmXSigto\n" @@ -260,7 +263,8 @@ static const char EX_EI_ED_MISSING_CERT[] = "\n" "\n" ; -const char EX_EI_ED_MISSING_CERT_FP[] = "E88E43E86015345A323D93D825C33E4AD1028F65"; +static const char EX_EI_ED_MISSING_CERT_FP[] = + "E88E43E86015345A323D93D825C33E4AD1028F65"; static const char EX_EI_ED_MISSING_CERT_KEY[] = "-----BEGIN RSA PUBLIC KEY-----\n" "MIGJAoGBALjA/geb0TR9rp/UPvLhABQpB0XUDYuZAnLkrv+i7AAV7FemTDveEGnc\n" @@ -284,7 +288,8 @@ static const char EX_EI_ED_BAD_CERT1[] = "-----END SIGNATURE-----\n" "\n" ; -const char EX_EI_ED_BAD_CERT1_FP[] = "F78D8A655607D32281D02144817A4F1D26AE520F"; +static const char EX_EI_ED_BAD_CERT1_FP[] = + "F78D8A655607D32281D02144817A4F1D26AE520F"; static const char EX_EI_ED_BAD_CERT1_KEY[] = "-----BEGIN RSA PUBLIC KEY-----\n" "MIGJAoGBAMlR46JhxsCmWYtmIB/JjTV2TUYIhJLmHy+X7FfkK3ZVQvvl9/3GSXFL\n" @@ -309,7 +314,8 @@ static const char EX_EI_ED_BAD_CERT2[] = "cVrtU6RVmzldSbyir8V/Z4S/Cm67gYAgjM5gfoFUqDs=\n" "-----END SIGNATURE-----\n" ; -const char EX_EI_ED_BAD_CERT2_FP[] = "7C2B42E783C4E0EB0CC3BDB37385D16737BACFBD"; +static const char EX_EI_ED_BAD_CERT2_FP[] = + "7C2B42E783C4E0EB0CC3BDB37385D16737BACFBD"; static const char EX_EI_ED_BAD_CERT2_KEY[] = "-----BEGIN RSA PUBLIC KEY-----\n" "MIGJAoGBALAM1F/0XJEsbxIQqb3+ObX/yGVnq9of8Q9sLsmxffD6hwVpCqnV3lTg\n" @@ -335,7 +341,8 @@ static const char EX_EI_ED_BAD_SIG1[] = "-----END SIGNATURE-----\n" "\n" ; -const char EX_EI_ED_BAD_SIG1_FP[] = "5AC3A538FEEFC6F9FCC5FA0CE64704396C30D62A"; +static const char EX_EI_ED_BAD_SIG1_FP[] = + "5AC3A538FEEFC6F9FCC5FA0CE64704396C30D62A"; static const char EX_EI_ED_BAD_SIG1_KEY[] = "-----BEGIN RSA PUBLIC KEY-----\n" "MIGJAoGBAMvb6SuoIkPfBkJgQuo5aQDepAs1kEETZ9VXotMlhB0JJikrqBrAAz+7\n" @@ -361,7 +368,8 @@ static const char EX_EI_ED_BAD_SIG2[] = "-----END SIGNATURE-----\n" "\n" ; -const char EX_EI_ED_BAD_SIG2_FP[] = "7F1D4DD477E340C6D6B389FAC26EDC746113082F"; +static const char EX_EI_ED_BAD_SIG2_FP[] = + "7F1D4DD477E340C6D6B389FAC26EDC746113082F"; static const char EX_EI_ED_BAD_SIG2_KEY[] = "-----BEGIN RSA PUBLIC KEY-----\n" "MIGJAoGBALzOyfCEUZnvCyhlyMctPkdXg/XRE3Cr6QgyzdKf5kQbUiu2n0FgSHOX\n" @@ -388,7 +396,8 @@ static const char EX_EI_ED_MISPLACED_CERT[] = "-----END SIGNATURE-----\n" "\n" ; -const char EX_EI_ED_MISPLACED_CERT_FP[] = "3B788BD0CE348BC5CED48313307C78175EB6D0F3"; +static const char EX_EI_ED_MISPLACED_CERT_FP[] = + "3B788BD0CE348BC5CED48313307C78175EB6D0F3"; static const char EX_EI_ED_MISPLACED_CERT_KEY[] = "-----BEGIN RSA PUBLIC KEY-----\n" "MIGJAoGBALTwNqhTprg1oC6bEbDqwIYBoER6prqUXQFbwbFDn+ekXhZj8vltgGwp\n" @@ -414,7 +423,8 @@ static const char EX_EI_ED_MISPLACED_SIG[] = "-----END SIGNATURE-----\n" "\n" ; -const char EX_EI_ED_MISPLACED_SIG_FP[] = "384E40A5DEED4AB1D8A74F1FCBDB18B7C24A8284"; +static const char EX_EI_ED_MISPLACED_SIG_FP[] = + "384E40A5DEED4AB1D8A74F1FCBDB18B7C24A8284"; static const char EX_EI_ED_MISPLACED_SIG_KEY[] = "-----BEGIN RSA PUBLIC KEY-----\n" "MIGJAoGBAK0HgOCG/6433VCrwz/vhk3cKmyOfenCp0GZ4DIUwPWt4DeyP4nTbN6T\n" diff --git a/src/test/include.am b/src/test/include.am index 7d80fdf152..d0bc808877 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -9,6 +9,12 @@ TESTS_ENVIRONMENT = \ export TESTING_TOR_BINARY="$(TESTING_TOR_BINARY)"; TESTSCRIPTS = src/test/test_zero_length_keys.sh \ + src/test/test_workqueue_cancel.sh \ + src/test/test_workqueue_efd.sh \ + src/test/test_workqueue_efd2.sh \ + src/test/test_workqueue_pipe.sh \ + src/test/test_workqueue_pipe2.sh \ + src/test/test_workqueue_socketpair.sh \ src/test/test_switch_id.sh if USEPYTHON @@ -16,7 +22,9 @@ TESTSCRIPTS += src/test/test_ntor.sh src/test/test_bt.sh endif TESTS += src/test/test src/test/test-slow src/test/test-memwipe \ - src/test/test_workqueue src/test/test_keygen.sh \ + src/test/test_workqueue \ + src/test/test_keygen.sh \ + src/test/test-timers \ $(TESTSCRIPTS) # These flavors are run using automake's test-driver and test-network.sh @@ -40,7 +48,8 @@ noinst_PROGRAMS+= \ src/test/test-memwipe \ src/test/test-child \ src/test/test_workqueue \ - src/test/test-switch-id + src/test/test-switch-id \ + src/test/test-timers endif src_test_AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ @@ -86,6 +95,7 @@ src_test_test_SOURCES = \ src/test/test_guardfraction.c \ src/test/test_extorport.c \ src/test/test_hs.c \ + src/test/test_handles.c \ src/test/test_introduce.c \ src/test/test_keypin.c \ src/test/test_link_handshake.c \ @@ -97,6 +107,7 @@ src_test_test_SOURCES = \ src/test/test_policy.c \ src/test/test_procmon.c \ src/test/test_pt.c \ + src/test/test_pubsub.c \ src/test/test_relay.c \ src/test/test_relaycell.c \ src/test/test_rendcache.c \ @@ -105,6 +116,7 @@ src_test_test_SOURCES = \ src/test/test_routerlist.c \ src/test/test_routerset.c \ src/test/test_scheduler.c \ + src/test/test_shared_random.c \ src/test/test_socks.c \ src/test/test_status.c \ src/test/test_threads.c \ @@ -127,6 +139,8 @@ src_test_test_slow_SOURCES = \ src_test_test_memwipe_SOURCES = \ src/test/test-memwipe.c +src_test_test_timers_SOURCES = \ + src/test/test-timers.c src_test_test_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) @@ -147,6 +161,7 @@ src_test_test_switch_id_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) src_test_test_switch_id_LDFLAGS = @TOR_LDFLAGS_zlib@ src_test_test_switch_id_LDADD = \ src/common/libor-testing.a \ + src/common/libor-ctime-testing.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ src_test_test_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @@ -156,6 +171,7 @@ src_test_test_LDADD = src/or/libtor-testing.a \ $(LIBKECCAK_TINY) \ $(LIBDONNA) \ src/common/libor-testing.a \ + src/common/libor-ctime-testing.a \ src/common/libor-event-testing.a \ src/trunnel/libor-trunnel-testing.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ @@ -168,13 +184,17 @@ src_test_test_slow_LDADD = $(src_test_test_LDADD) src_test_test_slow_LDFLAGS = $(src_test_test_LDFLAGS) src_test_test_memwipe_CPPFLAGS = $(src_test_test_CPPFLAGS) -src_test_test_memwipe_CFLAGS = $(src_test_test_CFLAGS) +# Don't use bugtrap cflags here: memwipe tests require memory violations. +src_test_test_memwipe_CFLAGS = $(TEST_CFLAGS) src_test_test_memwipe_LDADD = $(src_test_test_LDADD) -src_test_test_memwipe_LDFLAGS = $(src_test_test_LDFLAGS) +# The LDFLAGS need to include the bugtrap cflags, or else we won't link +# successfully with the libraries built with them. +src_test_test_memwipe_LDFLAGS = $(src_test_test_LDFLAGS) @CFLAGS_BUGTRAP@ src_test_bench_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @TOR_LDFLAGS_libevent@ src_test_bench_LDADD = src/or/libtor.a src/common/libor.a \ + src/common/libor-ctime.a \ src/common/libor-crypto.a $(LIBKECCAK_TINY) $(LIBDONNA) \ src/common/libor-event.a src/trunnel/libor-trunnel.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ @@ -185,11 +205,23 @@ src_test_test_workqueue_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ \ @TOR_LDFLAGS_libevent@ src_test_test_workqueue_LDADD = src/or/libtor-testing.a \ src/common/libor-testing.a \ + src/common/libor-ctime-testing.a \ src/common/libor-crypto-testing.a $(LIBKECCAK_TINY) $(LIBDONNA) \ src/common/libor-event-testing.a \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ +src_test_test_timers_CPPFLAGS = $(src_test_test_CPPFLAGS) +src_test_test_timers_CFLAGS = $(src_test_test_CFLAGS) +src_test_test_timers_LDADD = \ + src/common/libor-testing.a \ + src/common/libor-ctime-testing.a \ + src/common/libor-event-testing.a \ + src/common/libor-crypto-testing.a $(LIBKECCAK_TINY) $(LIBDONNA) \ + @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \ + @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ +src_test_test_timers_LDFLAGS = $(src_test_test_LDFLAGS) + noinst_HEADERS+= \ src/test/fakechans.h \ src/test/log_test_helpers.h \ @@ -208,6 +240,7 @@ noinst_PROGRAMS+= src/test/test-ntor-cl src_test_test_ntor_cl_SOURCES = src/test/test_ntor_cl.c src_test_test_ntor_cl_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ src_test_test_ntor_cl_LDADD = src/or/libtor.a src/common/libor.a \ + src/common/libor-ctime.a \ src/common/libor-crypto.a $(LIBKECCAK_TINY) $(LIBDONNA) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ @@ -217,6 +250,7 @@ src_test_test_ntor_cl_AM_CPPFLAGS = \ noinst_PROGRAMS += src/test/test-bt-cl src_test_test_bt_cl_SOURCES = src/test/test_bt_cl.c src_test_test_bt_cl_LDADD = src/common/libor-testing.a \ + src/common/libor-ctime-testing.a \ @TOR_LIB_MATH@ \ @TOR_LIB_WS32@ @TOR_LIB_GDI@ src_test_test_bt_cl_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) @@ -228,7 +262,14 @@ EXTRA_DIST += \ src/test/slownacl_curve25519.py \ src/test/zero_length_keys.sh \ src/test/test_keygen.sh \ - src/test/test_zero_length_keys.sh \ - src/test/test_ntor.sh src/test/test_bt.sh \ + src/test/test_zero_length_keys.sh \ + src/test/test_ntor.sh src/test/test_bt.sh \ src/test/test-network.sh \ - src/test/test_switch_id.sh + src/test/test_switch_id.sh \ + src/test/test_workqueue_cancel.sh \ + src/test/test_workqueue_efd.sh \ + src/test/test_workqueue_efd2.sh \ + src/test/test_workqueue_pipe.sh \ + src/test/test_workqueue_pipe2.sh \ + src/test/test_workqueue_socketpair.sh + diff --git a/src/test/sr_srv_calc_ref.py b/src/test/sr_srv_calc_ref.py new file mode 100644 index 0000000000..492ca62b15 --- /dev/null +++ b/src/test/sr_srv_calc_ref.py @@ -0,0 +1,71 @@ +# This is a reference implementation of the SRV calculation for prop250. We +# use it to generate a test vector for the test_sr_compute_srv() unittest. +# (./test shared-random/sr_compute_srv) +# +# Here is the SRV computation formula: +# +# HASHED_REVEALS = H(ID_a | R_a | ID_b | R_b | ..) +# +# SRV = SHA3-256("shared-random" | INT_8(reveal_num) | INT_4(version) | +# HASHED_REVEALS | previous_SRV) +# + +import sys +import hashlib +import struct + +# Python 3.6+, the SHA3 is available in hashlib natively. Else this requires +# the pysha3 package (pip install pysha3). +if sys.version_info < (3, 6): + import sha3 + +# Test vector to make sure the right sha3 version will be used. pysha3 < 1.0 +# used the old Keccak implementation. During the finalization of SHA3, NIST +# changed the delimiter suffix from 0x01 to 0x06. The Keccak sponge function +# stayed the same. pysha3 1.0 provides the previous Keccak hash, too. +TEST_VALUE = "e167f68d6563d75bb25f3aa49c29ef612d41352dc00606de7cbd630bb2665f51" +if TEST_VALUE != sha3.sha3_256(b"Hello World").hexdigest(): + print("pysha3 version is < 1.0. Please install from:") + print("https://github.com/tiran/pysha3https://github.com/tiran/pysha3") + sys.exit(1) + +# In this example, we use three reveal values. +reveal_num = 3 +version = 1 + +# We set directly the ascii value because memset(buf, 'A', 20) makes it to 20 +# times "41" in the final string. + +# Identity and reveal value of dirauth a +ID_a = 20 * "41" # RSA identity of 40 base16 bytes. +R_a = 56 * 'A' # 56 base64 characters + +# Identity and reveal value of dirauth b +ID_b = 20 * "42" # RSA identity of 40 base16 bytes. +R_b = 56 * 'B' # 56 base64 characters + +# Identity and reveal value of dirauth c +ID_c = 20 * "43" # RSA identity of 40 base16 bytes. +R_c = 56 * 'C' # 56 base64 characters + +# Concatenate them all together and hash them to form HASHED_REVEALS. +REVEALS = (ID_a + R_a + ID_b + R_b + ID_c + R_c).encode() +hashed_reveals_object = hashlib.sha3_256(REVEALS) +hashed_reveals = hashed_reveals_object.digest() + +previous_SRV = (32 * 'Z').encode() + +# Now form the message. +#srv_msg = struct.pack('13sQL256ss', "shared-random", reveal_num, version, +# hashed_reveals, previous_SRV) +invariant_token = b"shared-random" +srv_msg = invariant_token + \ + struct.pack('!QL', reveal_num, version) + \ + hashed_reveals + \ + previous_SRV + +# Now calculate the HMAC +srv = hashlib.sha3_256(srv_msg) +print("%s" % srv.hexdigest().upper()) + +# 2A9B1D6237DAB312A40F575DA85C147663E7ED3F80E9555395F15B515C74253D diff --git a/src/test/test-memwipe.c b/src/test/test-memwipe.c index 5d4fcec664..c28d5054a2 100644 --- a/src/test/test-memwipe.c +++ b/src/test/test-memwipe.c @@ -6,9 +6,6 @@ #include "crypto.h" #include "compat.h" -#undef MIN -#define MIN(a,b) ( ((a)<(b)) ? (a) : (b) ) - static unsigned fill_a_buffer_memset(void) __attribute__((noinline)); static unsigned fill_a_buffer_memwipe(void) __attribute__((noinline)); static unsigned fill_a_buffer_nothing(void) __attribute__((noinline)); @@ -17,6 +14,7 @@ static unsigned fill_heap_buffer_memwipe(void) __attribute__((noinline)); static unsigned fill_heap_buffer_nothing(void) __attribute__((noinline)); static unsigned check_a_buffer(void) __attribute__((noinline)); +extern const char *s; /* Make the linkage global */ const char *s = NULL; #define BUF_LEN 2048 diff --git a/src/test/test-timers.c b/src/test/test-timers.c new file mode 100644 index 0000000000..0196ec1fef --- /dev/null +++ b/src/test/test-timers.c @@ -0,0 +1,134 @@ +/* Copyright 2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" + +#include <math.h> +#include <stdio.h> +#include <string.h> + +#ifdef HAVE_EVENT2_EVENT_H +#include <event2/event.h> +#else +#include <event.h> +#endif + +#include "compat.h" +#include "compat_libevent.h" +#include "crypto.h" +#include "timers.h" +#include "util.h" + +#define N_TIMERS 1000 +#define MAX_DURATION 30 +#define N_DISABLE 5 + +static struct timeval fire_at[N_TIMERS] = { {0,0} }; +static int is_disabled[N_TIMERS] = {0}; +static int fired[N_TIMERS] = {0}; +static struct timeval difference[N_TIMERS] = { {0,0} }; +static tor_timer_t *timers[N_TIMERS] = {NULL}; + +static int n_active_timers = 0; +static int n_fired = 0; + +static void +timer_cb(tor_timer_t *t, void *arg, const struct timeval *now) +{ + tor_timer_t **t_ptr = arg; + tor_assert(*t_ptr == t); + int idx = (int) (t_ptr - timers); + ++fired[idx]; + timersub(now, &fire_at[idx], &difference[idx]); + ++n_fired; + // printf("%d / %d\n",n_fired, N_TIMERS); + if (n_fired == n_active_timers) { + event_base_loopbreak(tor_libevent_get_base()); + } +} + +int +main(int argc, char **argv) +{ + (void)argc; + (void)argv; + tor_libevent_cfg cfg; + memset(&cfg, 0, sizeof(cfg)); + tor_libevent_initialize(&cfg); + timers_initialize(); + + int i; + int ret; + struct timeval now; + tor_gettimeofday(&now); + for (i = 0; i < N_TIMERS; ++i) { + struct timeval delay; + delay.tv_sec = crypto_rand_int_range(0,MAX_DURATION); + delay.tv_usec = crypto_rand_int_range(0,1000000); + timeradd(&now, &delay, &fire_at[i]); + timers[i] = timer_new(timer_cb, &timers[i]); + timer_schedule(timers[i], &delay); + ++n_active_timers; + } + + /* Disable some; we'll make sure they don't trigger. */ + for (i = 0; i < N_DISABLE; ++i) { + int idx = crypto_rand_int_range(0, N_TIMERS); + if (is_disabled[idx]) + continue; + is_disabled[idx] = 1; + timer_disable(timers[idx]); + --n_active_timers; + } + + event_base_loop(tor_libevent_get_base(), 0); + + int64_t total_difference = 0; + uint64_t total_square_difference = 0; + tor_assert(n_fired == n_active_timers); + for (i = 0; i < N_TIMERS; ++i) { + if (is_disabled[i]) { + tor_assert(fired[i] == 0); + continue; + } + tor_assert(fired[i] == 1); + int64_t diff = difference[i].tv_usec + difference[i].tv_sec * 1000000; + total_difference += diff; + total_square_difference += diff*diff; + } + const int64_t mean_diff = total_difference / n_active_timers; + printf("mean difference: "U64_FORMAT" usec\n", + U64_PRINTF_ARG(mean_diff)); + + const double mean_sq = ((double)total_square_difference)/ n_active_timers; + const double sq_mean = mean_diff * mean_diff; + const double stddev = sqrt(mean_sq - sq_mean); + printf("standard deviation: %lf usec\n", stddev); + +#define MAX_DIFF_USEC (500*1000) +#define MAX_STDDEV_USEC (500*1000) +#define ODD_DIFF_USEC (2000) +#define ODD_STDDEV_USEC (2000) + + if (mean_diff < 0 || mean_diff > MAX_DIFF_USEC || stddev > MAX_STDDEV_USEC) { + printf("Either your system is under ridiculous load, or the " + "timer backend is broken.\n"); + ret = 1; + } else if (mean_diff > ODD_DIFF_USEC || stddev > ODD_STDDEV_USEC) { + printf("Either your system is a bit slow or the " + "timer backend is odd.\n"); + ret = 0; + } else { + printf("Looks good enough.\n"); + ret = 0; + } + + timer_free(NULL); + + for (i = 0; i < N_TIMERS; ++i) { + timer_free(timers[i]); + } + timers_shutdown(); + return ret; +} + diff --git a/src/test/test.c b/src/test/test.c index ed167a3e67..3a1054decf 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -1124,60 +1124,6 @@ static struct testcase_t test_array[] = { END_OF_TESTCASES }; -extern struct testcase_t accounting_tests[]; -extern struct testcase_t addr_tests[]; -extern struct testcase_t address_tests[]; -extern struct testcase_t buffer_tests[]; -extern struct testcase_t cell_format_tests[]; -extern struct testcase_t cell_queue_tests[]; -extern struct testcase_t channel_tests[]; -extern struct testcase_t channeltls_tests[]; -extern struct testcase_t checkdir_tests[]; -extern struct testcase_t circuitlist_tests[]; -extern struct testcase_t circuitmux_tests[]; -extern struct testcase_t compat_libevent_tests[]; -extern struct testcase_t config_tests[]; -extern struct testcase_t connection_tests[]; -extern struct testcase_t container_tests[]; -extern struct testcase_t controller_tests[]; -extern struct testcase_t controller_event_tests[]; -extern struct testcase_t crypto_tests[]; -extern struct testcase_t dir_tests[]; -extern struct testcase_t dir_handle_get_tests[]; -extern struct testcase_t entryconn_tests[]; -extern struct testcase_t entrynodes_tests[]; -extern struct testcase_t guardfraction_tests[]; -extern struct testcase_t extorport_tests[]; -extern struct testcase_t hs_tests[]; -extern struct testcase_t introduce_tests[]; -extern struct testcase_t keypin_tests[]; -extern struct testcase_t link_handshake_tests[]; -extern struct testcase_t logging_tests[]; -extern struct testcase_t microdesc_tests[]; -extern struct testcase_t nodelist_tests[]; -extern struct testcase_t oom_tests[]; -extern struct testcase_t options_tests[]; -extern struct testcase_t policy_tests[]; -extern struct testcase_t procmon_tests[]; -extern struct testcase_t pt_tests[]; -extern struct testcase_t relay_tests[]; -extern struct testcase_t relaycell_tests[]; -extern struct testcase_t rend_cache_tests[]; -extern struct testcase_t replaycache_tests[]; -extern struct testcase_t router_tests[]; -extern struct testcase_t routerkeys_tests[]; -extern struct testcase_t routerlist_tests[]; -extern struct testcase_t routerset_tests[]; -extern struct testcase_t scheduler_tests[]; -extern struct testcase_t socks_tests[]; -extern struct testcase_t status_tests[]; -extern struct testcase_t thread_tests[]; -extern struct testcase_t tortls_tests[]; -extern struct testcase_t util_tests[]; -extern struct testcase_t util_format_tests[]; -extern struct testcase_t util_process_tests[]; -extern struct testcase_t dns_tests[]; - struct testgroup_t testgroups[] = { { "", test_array }, { "accounting/", accounting_tests }, @@ -1224,13 +1170,16 @@ struct testgroup_t testgroups[] = { { "routerset/" , routerset_tests }, { "scheduler/", scheduler_tests }, { "socks/", socks_tests }, + { "shared-random/", sr_tests }, { "status/" , status_tests }, { "tortls/", tortls_tests }, { "util/", util_tests }, { "util/format/", util_format_tests }, { "util/logging/", logging_tests }, { "util/process/", util_process_tests }, + { "util/pubsub/", pubsub_tests }, { "util/thread/", thread_tests }, + { "util/handle/", handle_tests }, { "dns/", dns_tests }, END_OF_GROUPS }; diff --git a/src/test/test.h b/src/test/test.h index e618ce1224..6744d255f1 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -73,7 +73,7 @@ {print_ = (I64_PRINTF_TYPE) value_;}, {}, TT_EXIT_TEST_FUNCTION) const char *get_fname(const char *name); -crypto_pk_t *pk_generate(int idx); +struct crypto_pk_t *pk_generate(int idx); #define US2_CONCAT_2__(a, b) a ## __ ## b #define US_CONCAT_2__(a, b) a ## _ ## b @@ -163,11 +163,87 @@ crypto_pk_t *pk_generate(int idx); #define CALLED(mock_name) US_CONCAT_2_(NS(mock_name), called) #define NS_DECL(retval, mock_fn, args) \ + extern int CALLED(mock_fn); \ static retval NS(mock_fn) args; int CALLED(mock_fn) = 0 #define NS_MOCK(name) MOCK(name, NS(name)) #define NS_UNMOCK(name) UNMOCK(name) extern const struct testcase_setup_t passthrough_setup; +extern struct testcase_t accounting_tests[]; +extern struct testcase_t addr_tests[]; +extern struct testcase_t address_tests[]; +extern struct testcase_t buffer_tests[]; +extern struct testcase_t cell_format_tests[]; +extern struct testcase_t cell_queue_tests[]; +extern struct testcase_t channel_tests[]; +extern struct testcase_t channeltls_tests[]; +extern struct testcase_t checkdir_tests[]; +extern struct testcase_t circuitlist_tests[]; +extern struct testcase_t circuitmux_tests[]; +extern struct testcase_t compat_libevent_tests[]; +extern struct testcase_t config_tests[]; +extern struct testcase_t connection_tests[]; +extern struct testcase_t container_tests[]; +extern struct testcase_t controller_tests[]; +extern struct testcase_t controller_event_tests[]; +extern struct testcase_t crypto_tests[]; +extern struct testcase_t dir_tests[]; +extern struct testcase_t dir_handle_get_tests[]; +extern struct testcase_t entryconn_tests[]; +extern struct testcase_t entrynodes_tests[]; +extern struct testcase_t guardfraction_tests[]; +extern struct testcase_t extorport_tests[]; +extern struct testcase_t hs_tests[]; +extern struct testcase_t introduce_tests[]; +extern struct testcase_t keypin_tests[]; +extern struct testcase_t link_handshake_tests[]; +extern struct testcase_t logging_tests[]; +extern struct testcase_t microdesc_tests[]; +extern struct testcase_t nodelist_tests[]; +extern struct testcase_t oom_tests[]; +extern struct testcase_t options_tests[]; +extern struct testcase_t policy_tests[]; +extern struct testcase_t procmon_tests[]; +extern struct testcase_t pubsub_tests[]; +extern struct testcase_t pt_tests[]; +extern struct testcase_t relay_tests[]; +extern struct testcase_t relaycell_tests[]; +extern struct testcase_t rend_cache_tests[]; +extern struct testcase_t replaycache_tests[]; +extern struct testcase_t router_tests[]; +extern struct testcase_t routerkeys_tests[]; +extern struct testcase_t routerlist_tests[]; +extern struct testcase_t routerset_tests[]; +extern struct testcase_t scheduler_tests[]; +extern struct testcase_t socks_tests[]; +extern struct testcase_t status_tests[]; +extern struct testcase_t thread_tests[]; +extern struct testcase_t tortls_tests[]; +extern struct testcase_t util_tests[]; +extern struct testcase_t util_format_tests[]; +extern struct testcase_t util_process_tests[]; +extern struct testcase_t dns_tests[]; +extern struct testcase_t handle_tests[]; +extern struct testcase_t sr_tests[]; + +extern struct testcase_t slow_crypto_tests[]; +extern struct testcase_t slow_util_tests[]; + +extern struct testgroup_t testgroups[]; + +extern const char AUTHORITY_CERT_1[]; +extern const char AUTHORITY_SIGNKEY_1[]; +extern const char AUTHORITY_SIGNKEY_A_DIGEST[]; +extern const char AUTHORITY_SIGNKEY_A_DIGEST256[]; +extern const char AUTHORITY_CERT_2[]; +extern const char AUTHORITY_SIGNKEY_2[]; +extern const char AUTHORITY_SIGNKEY_B_DIGEST[]; +extern const char AUTHORITY_SIGNKEY_B_DIGEST256[]; +extern const char AUTHORITY_CERT_3[]; +extern const char AUTHORITY_SIGNKEY_3[]; +extern const char AUTHORITY_SIGNKEY_C_DIGEST[]; +extern const char AUTHORITY_SIGNKEY_C_DIGEST256[]; + #endif diff --git a/src/test/test_bt.sh b/src/test/test_bt.sh index 033acac955..312905a4e2 100755 --- a/src/test/test_bt.sh +++ b/src/test/test_bt.sh @@ -3,8 +3,11 @@ exitcode=0 +export ASAN_OPTIONS="handle_segv=0:allow_user_segv_handler=1" "${builddir:-.}/src/test/test-bt-cl" backtraces || exit $? -"${builddir:-.}/src/test/test-bt-cl" assert | "${PYTHON:-python}" "${abs_top_srcdir:-.}/src/test/bt_test.py" || exitcode="$?" -"${builddir:-.}/src/test/test-bt-cl" crash | "${PYTHON:-python}" "${abs_top_srcdir:-.}/src/test/bt_test.py" || exitcode="$?" +"${builddir:-.}/src/test/test-bt-cl" assert 2>&1 | "${PYTHON:-python}" "${abs_top_srcdir:-.}/src/test/bt_test.py" || exitcode="$?" +"${builddir:-.}/src/test/test-bt-cl" crash 2>&1 | "${PYTHON:-python}" "${abs_top_srcdir:-.}/src/test/bt_test.py" || exitcode="$?" + +"${builddir:-.}/src/test/test-bt-cl" none || exitcode="$?" exit ${exitcode} diff --git a/src/test/test_bt_cl.c b/src/test/test_bt_cl.c index 2f5e50fbf5..95b4f48f11 100644 --- a/src/test/test_bt_cl.c +++ b/src/test/test_bt_cl.c @@ -28,6 +28,9 @@ int a_tangled_web(int x) NOINLINE; int we_weave(int x) NOINLINE; static void abort_handler(int s) NORETURN; +#ifdef HAVE_CFLAG_WNULL_DEREFERENCE +DISABLE_GCC_WARNING(null-dereference) +#endif int crash(int x) { @@ -47,6 +50,9 @@ crash(int x) crashtype *= x; return crashtype; } +#ifdef HAVE_CFLAG_WNULL_DEREFERENCE +ENABLE_GCC_WARNING(null-dereference) +#endif int oh_what(int x) diff --git a/src/test/test_buffers.c b/src/test/test_buffers.c index e5e56edf75..95584d54a8 100644 --- a/src/test/test_buffers.c +++ b/src/test/test_buffers.c @@ -695,9 +695,9 @@ test_buffers_zlib_fin_at_chunk_end(void *arg) tor_free(msg); } -const uint8_t *tls_read_ptr; -int n_remaining; -int next_reply_val[16]; +static const uint8_t *tls_read_ptr; +static int n_remaining; +static int next_reply_val[16]; static int mock_tls_read(tor_tls_t *tls, char *cp, size_t len) diff --git a/src/test/test_channel.c b/src/test/test_channel.c index 846e419fea..a9e0634d9e 100644 --- a/src/test/test_channel.c +++ b/src/test/test_channel.c @@ -20,9 +20,6 @@ #include "test.h" #include "fakechans.h" -/* This comes from channel.c */ -extern uint64_t estimated_total_queue_size; - static int test_chan_accept_cells = 0; static int test_chan_fixed_cells_recved = 0; static cell_t * test_chan_last_seen_fixed_cell_ptr = NULL; @@ -33,7 +30,7 @@ static int test_destroy_not_pending_calls = 0; static int test_doesnt_want_writes_count = 0; static int test_dumpstats_calls = 0; static int test_has_waiting_cells_count = 0; -static double test_overhead_estimate = 1.0f; +static double test_overhead_estimate = 1.0; static int test_releases_count = 0; static circuitmux_t *test_target_cmux = NULL; static unsigned int test_cmux_cells = 0; @@ -792,7 +789,7 @@ test_channel_incoming(void *arg) /* Accept cells to lower layer */ test_chan_accept_cells = 1; /* Use default overhead factor */ - test_overhead_estimate = 1.0f; + test_overhead_estimate = 1.0; ch = new_fake_channel(); tt_assert(ch); @@ -881,7 +878,7 @@ test_channel_lifecycle(void *arg) /* Accept cells to lower layer */ test_chan_accept_cells = 1; /* Use default overhead factor */ - test_overhead_estimate = 1.0f; + test_overhead_estimate = 1.0; ch1 = new_fake_channel(); tt_assert(ch1); @@ -989,7 +986,7 @@ test_channel_lifecycle_2(void *arg) /* Accept cells to lower layer */ test_chan_accept_cells = 1; /* Use default overhead factor */ - test_overhead_estimate = 1.0f; + test_overhead_estimate = 1.0; ch = new_fake_channel(); tt_assert(ch); @@ -1136,7 +1133,7 @@ test_channel_multi(void *arg) /* Accept cells to lower layer */ test_chan_accept_cells = 1; /* Use default overhead factor */ - test_overhead_estimate = 1.0f; + test_overhead_estimate = 1.0; ch1 = new_fake_channel(); tt_assert(ch1); @@ -1444,7 +1441,7 @@ test_channel_queue_incoming(void *arg) /* Accept cells to lower layer */ test_chan_accept_cells = 1; /* Use default overhead factor */ - test_overhead_estimate = 1.0f; + test_overhead_estimate = 1.0; ch = new_fake_channel(); tt_assert(ch); @@ -1584,16 +1581,16 @@ test_channel_queue_size(void *arg) /* One cell, times an overhead factor of 1.0 */ tt_u64_op(ch->bytes_queued_for_xmit, ==, 512); /* Try a different overhead factor */ - test_overhead_estimate = 0.5f; + test_overhead_estimate = 0.5; /* This one should be ignored since it's below 1.0 */ channel_update_xmit_queue_size(ch); tt_u64_op(ch->bytes_queued_for_xmit, ==, 512); /* Now try a larger one */ - test_overhead_estimate = 2.0f; + test_overhead_estimate = 2.0; channel_update_xmit_queue_size(ch); tt_u64_op(ch->bytes_queued_for_xmit, ==, 1024); /* Go back to 1.0 */ - test_overhead_estimate = 1.0f; + test_overhead_estimate = 1.0; channel_update_xmit_queue_size(ch); tt_u64_op(ch->bytes_queued_for_xmit, ==, 512); /* Check the global estimate too */ diff --git a/src/test/test_channeltls.c b/src/test/test_channeltls.c index 04ae9a6da7..394612f155 100644 --- a/src/test/test_channeltls.c +++ b/src/test/test_channeltls.c @@ -185,7 +185,7 @@ test_channeltls_overhead_estimate(void *arg) const char test_digest[DIGEST_LEN] = { 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14 }; - float r; + double r; channel_tls_t *tlschan = NULL; (void)arg; @@ -206,31 +206,31 @@ test_channeltls_overhead_estimate(void *arg) ch = channel_tls_connect(&test_addr, 567, test_digest); tt_assert(ch != NULL); - /* First case: silly low ratios should get clamped to 1.0f */ + /* First case: silly low ratios should get clamped to 1.0 */ tlschan = BASE_CHAN_TO_TLS(ch); tt_assert(tlschan != NULL); tlschan->conn->bytes_xmitted = 128; tlschan->conn->bytes_xmitted_by_tls = 64; r = ch->get_overhead_estimate(ch); - tt_assert(fabsf(r - 1.0f) < 1E-12); + tt_assert(fabs(r - 1.0) < 1E-12); tlschan->conn->bytes_xmitted_by_tls = 127; r = ch->get_overhead_estimate(ch); - tt_assert(fabsf(r - 1.0f) < 1E-12); + tt_assert(fabs(r - 1.0) < 1E-12); /* Now middle of the range */ tlschan->conn->bytes_xmitted_by_tls = 192; r = ch->get_overhead_estimate(ch); - tt_assert(fabsf(r - 1.5f) < 1E-12); + tt_assert(fabs(r - 1.5) < 1E-12); - /* Now above the 2.0f clamp */ + /* Now above the 2.0 clamp */ tlschan->conn->bytes_xmitted_by_tls = 257; r = ch->get_overhead_estimate(ch); - tt_assert(fabsf(r - 2.0f) < 1E-12); + tt_assert(fabs(r - 2.0) < 1E-12); tlschan->conn->bytes_xmitted_by_tls = 512; r = ch->get_overhead_estimate(ch); - tt_assert(fabsf(r - 2.0f) < 1E-12); + tt_assert(fabs(r - 2.0) < 1E-12); done: if (ch) { diff --git a/src/test/test_controller.c b/src/test/test_controller.c index 7f9db4312f..0dea8473b9 100644 --- a/src/test/test_controller.c +++ b/src/test/test_controller.c @@ -4,7 +4,10 @@ #define CONTROL_PRIVATE #include "or.h" #include "control.h" +#include "entrynodes.h" +#include "networkstatus.h" #include "rendservice.h" +#include "routerlist.h" #include "test.h" static void @@ -154,10 +157,1139 @@ test_rend_service_parse_port_config(void *arg) tor_free(err_msg); } +static void +test_add_onion_helper_clientauth(void *arg) +{ + rend_authorized_client_t *client = NULL; + char *err_msg = NULL; + int created = 0; + + (void)arg; + + /* Test "ClientName" only. */ + client = add_onion_helper_clientauth("alice", &created, &err_msg); + tt_assert(client); + tt_assert(created); + tt_assert(!err_msg); + rend_authorized_client_free(client); + + /* Test "ClientName:Blob" */ + client = add_onion_helper_clientauth("alice:475hGBHPlq7Mc0cRZitK/B", + &created, &err_msg); + tt_assert(client); + tt_assert(!created); + tt_assert(!err_msg); + rend_authorized_client_free(client); + + /* Test invalid client names */ + client = add_onion_helper_clientauth("no*asterisks*allowed", &created, + &err_msg); + tt_assert(!client); + tt_assert(err_msg); + tor_free(err_msg); + + /* Test invalid auth cookie */ + client = add_onion_helper_clientauth("alice:12345", &created, &err_msg); + tt_assert(!client); + tt_assert(err_msg); + tor_free(err_msg); + + /* Test invalid syntax */ + client = add_onion_helper_clientauth(":475hGBHPlq7Mc0cRZitK/B", &created, + &err_msg); + tt_assert(!client); + tt_assert(err_msg); + tor_free(err_msg); + + done: + rend_authorized_client_free(client); + tor_free(err_msg); +} + +/* Mocks and data/variables used for GETINFO download status tests */ + +static const download_status_t dl_status_default = + { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_ANY_DIRSERVER, + DL_SCHED_INCREMENT_FAILURE, DL_SCHED_RANDOM_EXPONENTIAL, 0, 0 }; +static download_status_t ns_dl_status[N_CONSENSUS_FLAVORS]; +static download_status_t ns_dl_status_bootstrap[N_CONSENSUS_FLAVORS]; +static download_status_t ns_dl_status_running[N_CONSENSUS_FLAVORS]; + +/* + * These should explore all the possible cases of download_status_to_string() + * in control.c + */ +static const download_status_t dls_sample_1 = + { 1467163900, 0, 0, DL_SCHED_GENERIC, DL_WANT_ANY_DIRSERVER, + DL_SCHED_INCREMENT_FAILURE, DL_SCHED_DETERMINISTIC, 0, 0 }; +static const char * dls_sample_1_str = + "next-attempt-at 2016-06-29 01:31:40\n" + "n-download-failures 0\n" + "n-download-attempts 0\n" + "schedule DL_SCHED_GENERIC\n" + "want-authority DL_WANT_ANY_DIRSERVER\n" + "increment-on DL_SCHED_INCREMENT_FAILURE\n" + "backoff DL_SCHED_DETERMINISTIC\n"; +static const download_status_t dls_sample_2 = + { 1467164400, 1, 2, DL_SCHED_CONSENSUS, DL_WANT_AUTHORITY, + DL_SCHED_INCREMENT_FAILURE, DL_SCHED_DETERMINISTIC, 0, 0 }; +static const char * dls_sample_2_str = + "next-attempt-at 2016-06-29 01:40:00\n" + "n-download-failures 1\n" + "n-download-attempts 2\n" + "schedule DL_SCHED_CONSENSUS\n" + "want-authority DL_WANT_AUTHORITY\n" + "increment-on DL_SCHED_INCREMENT_FAILURE\n" + "backoff DL_SCHED_DETERMINISTIC\n"; +static const download_status_t dls_sample_3 = + { 1467154400, 12, 25, DL_SCHED_BRIDGE, DL_WANT_ANY_DIRSERVER, + DL_SCHED_INCREMENT_ATTEMPT, DL_SCHED_DETERMINISTIC, 0, 0 }; +static const char * dls_sample_3_str = + "next-attempt-at 2016-06-28 22:53:20\n" + "n-download-failures 12\n" + "n-download-attempts 25\n" + "schedule DL_SCHED_BRIDGE\n" + "want-authority DL_WANT_ANY_DIRSERVER\n" + "increment-on DL_SCHED_INCREMENT_ATTEMPT\n" + "backoff DL_SCHED_DETERMINISTIC\n"; +static const download_status_t dls_sample_4 = + { 1467166600, 3, 0, DL_SCHED_GENERIC, DL_WANT_ANY_DIRSERVER, + DL_SCHED_INCREMENT_FAILURE, DL_SCHED_RANDOM_EXPONENTIAL, 0, 0 }; +static const char * dls_sample_4_str = + "next-attempt-at 2016-06-29 02:16:40\n" + "n-download-failures 3\n" + "n-download-attempts 0\n" + "schedule DL_SCHED_GENERIC\n" + "want-authority DL_WANT_ANY_DIRSERVER\n" + "increment-on DL_SCHED_INCREMENT_FAILURE\n" + "backoff DL_SCHED_RANDOM_EXPONENTIAL\n" + "last-backoff-position 0\n" + "last-delay-used 0\n"; +static const download_status_t dls_sample_5 = + { 1467164600, 3, 7, DL_SCHED_CONSENSUS, DL_WANT_ANY_DIRSERVER, + DL_SCHED_INCREMENT_FAILURE, DL_SCHED_RANDOM_EXPONENTIAL, 1, 2112, }; +static const char * dls_sample_5_str = + "next-attempt-at 2016-06-29 01:43:20\n" + "n-download-failures 3\n" + "n-download-attempts 7\n" + "schedule DL_SCHED_CONSENSUS\n" + "want-authority DL_WANT_ANY_DIRSERVER\n" + "increment-on DL_SCHED_INCREMENT_FAILURE\n" + "backoff DL_SCHED_RANDOM_EXPONENTIAL\n" + "last-backoff-position 1\n" + "last-delay-used 2112\n"; +static const download_status_t dls_sample_6 = + { 1467164200, 4, 9, DL_SCHED_CONSENSUS, DL_WANT_AUTHORITY, + DL_SCHED_INCREMENT_ATTEMPT, DL_SCHED_RANDOM_EXPONENTIAL, 3, 432 }; +static const char * dls_sample_6_str = + "next-attempt-at 2016-06-29 01:36:40\n" + "n-download-failures 4\n" + "n-download-attempts 9\n" + "schedule DL_SCHED_CONSENSUS\n" + "want-authority DL_WANT_AUTHORITY\n" + "increment-on DL_SCHED_INCREMENT_ATTEMPT\n" + "backoff DL_SCHED_RANDOM_EXPONENTIAL\n" + "last-backoff-position 3\n" + "last-delay-used 432\n"; + +/* Simulated auth certs */ +static const char *auth_id_digest_1_str = + "63CDD326DFEF0CA020BDD3FEB45A3286FE13A061"; +static download_status_t auth_def_cert_download_status_1; +static const char *auth_id_digest_2_str = + "2C209FCDD8D48DC049777B8DC2C0F94A0408BE99"; +static download_status_t auth_def_cert_download_status_2; +/* Expected form of digest list returned for GETINFO downloads/cert/fps */ +static const char *auth_id_digest_expected_list = + "63CDD326DFEF0CA020BDD3FEB45A3286FE13A061\n" + "2C209FCDD8D48DC049777B8DC2C0F94A0408BE99\n"; + +/* Signing keys for simulated auth 1 */ +static const char *auth_1_sk_1_str = + "AA69566029B1F023BA09451B8F1B10952384EB58"; +static download_status_t auth_1_sk_1_dls; +static const char *auth_1_sk_2_str = + "710865C7F06B73C5292695A8C34F1C94F769FF72"; +static download_status_t auth_1_sk_2_dls; +/* + * Expected form of sk digest list for + * GETINFO downloads/cert/<auth_id_digest_1_str>/sks + */ +static const char *auth_1_sk_digest_expected_list = + "AA69566029B1F023BA09451B8F1B10952384EB58\n" + "710865C7F06B73C5292695A8C34F1C94F769FF72\n"; + +/* Signing keys for simulated auth 2 */ +static const char *auth_2_sk_1_str = + "4299047E00D070AD6703FE00BE7AA756DB061E62"; +static download_status_t auth_2_sk_1_dls; +static const char *auth_2_sk_2_str = + "9451B8F1B10952384EB58B5F230C0BB701626C9B"; +static download_status_t auth_2_sk_2_dls; +/* + * Expected form of sk digest list for + * GETINFO downloads/cert/<auth_id_digest_2_str>/sks + */ +static const char *auth_2_sk_digest_expected_list = + "4299047E00D070AD6703FE00BE7AA756DB061E62\n" + "9451B8F1B10952384EB58B5F230C0BB701626C9B\n"; + +/* Simulated router descriptor digests or bridge identity digests */ +static const char *descbr_digest_1_str = + "616408544C7345822696074A1A3DFA16AB381CBD"; +static download_status_t descbr_digest_1_dl; +static const char *descbr_digest_2_str = + "06E8067246967265DBCB6641631B530EFEC12DC3"; +static download_status_t descbr_digest_2_dl; +/* Expected form of digest list returned for GETINFO downloads/desc/descs */ +static const char *descbr_expected_list = + "616408544C7345822696074A1A3DFA16AB381CBD\n" + "06E8067246967265DBCB6641631B530EFEC12DC3\n"; +/* + * Flag to make all descbr queries fail, to simulate not being + * configured such that such queries make sense. + */ +static int disable_descbr = 0; + +static void +reset_mocked_dl_statuses(void) +{ + int i; + + for (i = 0; i < N_CONSENSUS_FLAVORS; ++i) { + memcpy(&(ns_dl_status[i]), &dl_status_default, + sizeof(download_status_t)); + memcpy(&(ns_dl_status_bootstrap[i]), &dl_status_default, + sizeof(download_status_t)); + memcpy(&(ns_dl_status_running[i]), &dl_status_default, + sizeof(download_status_t)); + } + + memcpy(&auth_def_cert_download_status_1, &dl_status_default, + sizeof(download_status_t)); + memcpy(&auth_def_cert_download_status_2, &dl_status_default, + sizeof(download_status_t)); + memcpy(&auth_1_sk_1_dls, &dl_status_default, + sizeof(download_status_t)); + memcpy(&auth_1_sk_2_dls, &dl_status_default, + sizeof(download_status_t)); + memcpy(&auth_2_sk_1_dls, &dl_status_default, + sizeof(download_status_t)); + memcpy(&auth_2_sk_2_dls, &dl_status_default, + sizeof(download_status_t)); + + memcpy(&descbr_digest_1_dl, &dl_status_default, + sizeof(download_status_t)); + memcpy(&descbr_digest_2_dl, &dl_status_default, + sizeof(download_status_t)); +} + +static download_status_t * +ns_dl_status_mock(consensus_flavor_t flavor) +{ + return &(ns_dl_status[flavor]); +} + +static download_status_t * +ns_dl_status_bootstrap_mock(consensus_flavor_t flavor) +{ + return &(ns_dl_status_bootstrap[flavor]); +} + +static download_status_t * +ns_dl_status_running_mock(consensus_flavor_t flavor) +{ + return &(ns_dl_status_running[flavor]); +} + +static void +setup_ns_mocks(void) +{ + MOCK(networkstatus_get_dl_status_by_flavor, ns_dl_status_mock); + MOCK(networkstatus_get_dl_status_by_flavor_bootstrap, + ns_dl_status_bootstrap_mock); + MOCK(networkstatus_get_dl_status_by_flavor_running, + ns_dl_status_running_mock); + reset_mocked_dl_statuses(); +} + +static void +clear_ns_mocks(void) +{ + UNMOCK(networkstatus_get_dl_status_by_flavor); + UNMOCK(networkstatus_get_dl_status_by_flavor_bootstrap); + UNMOCK(networkstatus_get_dl_status_by_flavor_running); +} + +static smartlist_t * +cert_dl_status_auth_ids_mock(void) +{ + char digest[DIGEST_LEN], *tmp; + int len; + smartlist_t *list = NULL; + + /* Just pretend we have only the two hard-coded digests listed above */ + list = smartlist_new(); + len = base16_decode(digest, DIGEST_LEN, + auth_id_digest_1_str, strlen(auth_id_digest_1_str)); + tt_int_op(len, OP_EQ, DIGEST_LEN); + tmp = tor_malloc(DIGEST_LEN); + memcpy(tmp, digest, DIGEST_LEN); + smartlist_add(list, tmp); + len = base16_decode(digest, DIGEST_LEN, + auth_id_digest_2_str, strlen(auth_id_digest_2_str)); + tt_int_op(len, OP_EQ, DIGEST_LEN); + tmp = tor_malloc(DIGEST_LEN); + memcpy(tmp, digest, DIGEST_LEN); + smartlist_add(list, tmp); + + done: + return list; +} + +static download_status_t * +cert_dl_status_def_for_auth_mock(const char *digest) +{ + download_status_t *dl = NULL; + char digest_str[HEX_DIGEST_LEN+1]; + + tt_assert(digest != NULL); + base16_encode(digest_str, HEX_DIGEST_LEN + 1, + digest, DIGEST_LEN); + digest_str[HEX_DIGEST_LEN] = '\0'; + + if (strcmp(digest_str, auth_id_digest_1_str) == 0) { + dl = &auth_def_cert_download_status_1; + } else if (strcmp(digest_str, auth_id_digest_2_str) == 0) { + dl = &auth_def_cert_download_status_2; + } + + done: + return dl; +} + +static smartlist_t * +cert_dl_status_sks_for_auth_id_mock(const char *digest) +{ + smartlist_t *list = NULL; + char sk[DIGEST_LEN]; + char digest_str[HEX_DIGEST_LEN+1]; + char *tmp; + int len; + + tt_assert(digest != NULL); + base16_encode(digest_str, HEX_DIGEST_LEN + 1, + digest, DIGEST_LEN); + digest_str[HEX_DIGEST_LEN] = '\0'; + + /* + * Build a list of two hard-coded digests, depending on what we + * were just passed. + */ + if (strcmp(digest_str, auth_id_digest_1_str) == 0) { + list = smartlist_new(); + len = base16_decode(sk, DIGEST_LEN, + auth_1_sk_1_str, strlen(auth_1_sk_1_str)); + tt_int_op(len, OP_EQ, DIGEST_LEN); + tmp = tor_malloc(DIGEST_LEN); + memcpy(tmp, sk, DIGEST_LEN); + smartlist_add(list, tmp); + len = base16_decode(sk, DIGEST_LEN, + auth_1_sk_2_str, strlen(auth_1_sk_2_str)); + tt_int_op(len, OP_EQ, DIGEST_LEN); + tmp = tor_malloc(DIGEST_LEN); + memcpy(tmp, sk, DIGEST_LEN); + smartlist_add(list, tmp); + } else if (strcmp(digest_str, auth_id_digest_2_str) == 0) { + list = smartlist_new(); + len = base16_decode(sk, DIGEST_LEN, + auth_2_sk_1_str, strlen(auth_2_sk_1_str)); + tt_int_op(len, OP_EQ, DIGEST_LEN); + tmp = tor_malloc(DIGEST_LEN); + memcpy(tmp, sk, DIGEST_LEN); + smartlist_add(list, tmp); + len = base16_decode(sk, DIGEST_LEN, + auth_2_sk_2_str, strlen(auth_2_sk_2_str)); + tt_int_op(len, OP_EQ, DIGEST_LEN); + tmp = tor_malloc(DIGEST_LEN); + memcpy(tmp, sk, DIGEST_LEN); + smartlist_add(list, tmp); + } + + done: + return list; +} + +static download_status_t * +cert_dl_status_fp_sk_mock(const char *fp_digest, const char *sk_digest) +{ + download_status_t *dl = NULL; + char fp_digest_str[HEX_DIGEST_LEN+1], sk_digest_str[HEX_DIGEST_LEN+1]; + + /* + * Unpack the digests so we can compare them and figure out which + * dl status we want. + */ + + tt_assert(fp_digest != NULL); + base16_encode(fp_digest_str, HEX_DIGEST_LEN + 1, + fp_digest, DIGEST_LEN); + fp_digest_str[HEX_DIGEST_LEN] = '\0'; + tt_assert(sk_digest != NULL); + base16_encode(sk_digest_str, HEX_DIGEST_LEN + 1, + sk_digest, DIGEST_LEN); + sk_digest_str[HEX_DIGEST_LEN] = '\0'; + + if (strcmp(fp_digest_str, auth_id_digest_1_str) == 0) { + if (strcmp(sk_digest_str, auth_1_sk_1_str) == 0) { + dl = &auth_1_sk_1_dls; + } else if (strcmp(sk_digest_str, auth_1_sk_2_str) == 0) { + dl = &auth_1_sk_2_dls; + } + } else if (strcmp(fp_digest_str, auth_id_digest_2_str) == 0) { + if (strcmp(sk_digest_str, auth_2_sk_1_str) == 0) { + dl = &auth_2_sk_1_dls; + } else if (strcmp(sk_digest_str, auth_2_sk_2_str) == 0) { + dl = &auth_2_sk_2_dls; + } + } + + done: + return dl; +} + +static void +setup_cert_mocks(void) +{ + MOCK(list_authority_ids_with_downloads, cert_dl_status_auth_ids_mock); + MOCK(id_only_download_status_for_authority_id, + cert_dl_status_def_for_auth_mock); + MOCK(list_sk_digests_for_authority_id, + cert_dl_status_sks_for_auth_id_mock); + MOCK(download_status_for_authority_id_and_sk, + cert_dl_status_fp_sk_mock); + reset_mocked_dl_statuses(); +} + +static void +clear_cert_mocks(void) +{ + UNMOCK(list_authority_ids_with_downloads); + UNMOCK(id_only_download_status_for_authority_id); + UNMOCK(list_sk_digests_for_authority_id); + UNMOCK(download_status_for_authority_id_and_sk); +} + +static smartlist_t * +descbr_get_digests_mock(void) +{ + char digest[DIGEST_LEN], *tmp; + int len; + smartlist_t *list = NULL; + + if (!disable_descbr) { + /* Just pretend we have only the two hard-coded digests listed above */ + list = smartlist_new(); + len = base16_decode(digest, DIGEST_LEN, + descbr_digest_1_str, strlen(descbr_digest_1_str)); + tt_int_op(len, OP_EQ, DIGEST_LEN); + tmp = tor_malloc(DIGEST_LEN); + memcpy(tmp, digest, DIGEST_LEN); + smartlist_add(list, tmp); + len = base16_decode(digest, DIGEST_LEN, + descbr_digest_2_str, strlen(descbr_digest_2_str)); + tt_int_op(len, OP_EQ, DIGEST_LEN); + tmp = tor_malloc(DIGEST_LEN); + memcpy(tmp, digest, DIGEST_LEN); + smartlist_add(list, tmp); + } + + done: + return list; +} + +static download_status_t * +descbr_get_dl_by_digest_mock(const char *digest) +{ + download_status_t *dl = NULL; + char digest_str[HEX_DIGEST_LEN+1]; + + if (!disable_descbr) { + tt_assert(digest != NULL); + base16_encode(digest_str, HEX_DIGEST_LEN + 1, + digest, DIGEST_LEN); + digest_str[HEX_DIGEST_LEN] = '\0'; + + if (strcmp(digest_str, descbr_digest_1_str) == 0) { + dl = &descbr_digest_1_dl; + } else if (strcmp(digest_str, descbr_digest_2_str) == 0) { + dl = &descbr_digest_2_dl; + } + } + + done: + return dl; +} + +static void +setup_desc_mocks(void) +{ + MOCK(router_get_descriptor_digests, + descbr_get_digests_mock); + MOCK(router_get_dl_status_by_descriptor_digest, + descbr_get_dl_by_digest_mock); + reset_mocked_dl_statuses(); +} + +static void +clear_desc_mocks(void) +{ + UNMOCK(router_get_descriptor_digests); + UNMOCK(router_get_dl_status_by_descriptor_digest); +} + +static void +setup_bridge_mocks(void) +{ + disable_descbr = 0; + + MOCK(list_bridge_identities, + descbr_get_digests_mock); + MOCK(get_bridge_dl_status_by_id, + descbr_get_dl_by_digest_mock); + reset_mocked_dl_statuses(); +} + +static void +clear_bridge_mocks(void) +{ + UNMOCK(list_bridge_identities); + UNMOCK(get_bridge_dl_status_by_id); + + disable_descbr = 0; +} + +static void +test_download_status_consensus(void *arg) +{ + /* We just need one of these to pass, it doesn't matter what's in it */ + control_connection_t dummy; + /* Get results out */ + char *answer = NULL; + const char *errmsg = NULL; + + (void)arg; + + /* Check that the unknown prefix case works; no mocks needed yet */ + getinfo_helper_downloads(&dummy, "downloads/foo", &answer, &errmsg); + tt_assert(answer == NULL); + tt_str_op(errmsg, OP_EQ, "Unknown download status query"); + + setup_ns_mocks(); + + /* + * Check returning serialized dlstatuses, and implicitly also test + * download_status_to_string(). + */ + + /* Case 1 default/FLAV_NS*/ + memcpy(&(ns_dl_status[FLAV_NS]), &dls_sample_1, + sizeof(download_status_t)); + getinfo_helper_downloads(&dummy, "downloads/networkstatus/ns", + &answer, &errmsg); + tt_assert(answer != NULL); + tt_assert(errmsg == NULL); + tt_str_op(answer, OP_EQ, dls_sample_1_str); + tor_free(answer); + errmsg = NULL; + + /* Case 2 default/FLAV_MICRODESC */ + memcpy(&(ns_dl_status[FLAV_MICRODESC]), &dls_sample_2, + sizeof(download_status_t)); + getinfo_helper_downloads(&dummy, "downloads/networkstatus/microdesc", + &answer, &errmsg); + tt_assert(answer != NULL); + tt_assert(errmsg == NULL); + tt_str_op(answer, OP_EQ, dls_sample_2_str); + tor_free(answer); + errmsg = NULL; + + /* Case 3 bootstrap/FLAV_NS */ + memcpy(&(ns_dl_status_bootstrap[FLAV_NS]), &dls_sample_3, + sizeof(download_status_t)); + getinfo_helper_downloads(&dummy, "downloads/networkstatus/ns/bootstrap", + &answer, &errmsg); + tt_assert(answer != NULL); + tt_assert(errmsg == NULL); + tt_str_op(answer, OP_EQ, dls_sample_3_str); + tor_free(answer); + errmsg = NULL; + + /* Case 4 bootstrap/FLAV_MICRODESC */ + memcpy(&(ns_dl_status_bootstrap[FLAV_MICRODESC]), &dls_sample_4, + sizeof(download_status_t)); + getinfo_helper_downloads(&dummy, + "downloads/networkstatus/microdesc/bootstrap", + &answer, &errmsg); + tt_assert(answer != NULL); + tt_assert(errmsg == NULL); + tt_str_op(answer, OP_EQ, dls_sample_4_str); + tor_free(answer); + errmsg = NULL; + + /* Case 5 running/FLAV_NS */ + memcpy(&(ns_dl_status_running[FLAV_NS]), &dls_sample_5, + sizeof(download_status_t)); + getinfo_helper_downloads(&dummy, + "downloads/networkstatus/ns/running", + &answer, &errmsg); + tt_assert(answer != NULL); + tt_assert(errmsg == NULL); + tt_str_op(answer, OP_EQ, dls_sample_5_str); + tor_free(answer); + errmsg = NULL; + + /* Case 6 running/FLAV_MICRODESC */ + memcpy(&(ns_dl_status_running[FLAV_MICRODESC]), &dls_sample_6, + sizeof(download_status_t)); + getinfo_helper_downloads(&dummy, + "downloads/networkstatus/microdesc/running", + &answer, &errmsg); + tt_assert(answer != NULL); + tt_assert(errmsg == NULL); + tt_str_op(answer, OP_EQ, dls_sample_6_str); + tor_free(answer); + errmsg = NULL; + + /* Now check the error case */ + getinfo_helper_downloads(&dummy, "downloads/networkstatus/foo", + &answer, &errmsg); + tt_assert(answer == NULL); + tt_assert(errmsg != NULL); + tt_str_op(errmsg, OP_EQ, "Unknown flavor"); + errmsg = NULL; + + done: + clear_ns_mocks(); + tor_free(answer); + + return; +} + +static void +test_download_status_cert(void *arg) +{ + /* We just need one of these to pass, it doesn't matter what's in it */ + control_connection_t dummy; + /* Get results out */ + char *question = NULL; + char *answer = NULL; + const char *errmsg = NULL; + + (void)arg; + + setup_cert_mocks(); + + /* + * Check returning serialized dlstatuses and digest lists, and implicitly + * also test download_status_to_string() and digest_list_to_string(). + */ + + /* Case 1 - list of authority identity fingerprints */ + getinfo_helper_downloads(&dummy, + "downloads/cert/fps", + &answer, &errmsg); + tt_assert(answer != NULL); + tt_assert(errmsg == NULL); + tt_str_op(answer, OP_EQ, auth_id_digest_expected_list); + tor_free(answer); + errmsg = NULL; + + /* Case 2 - download status for default cert for 1st auth id */ + memcpy(&auth_def_cert_download_status_1, &dls_sample_1, + sizeof(download_status_t)); + tor_asprintf(&question, "downloads/cert/fp/%s", auth_id_digest_1_str); + tt_assert(question != NULL); + getinfo_helper_downloads(&dummy, question, &answer, &errmsg); + tt_assert(answer != NULL); + tt_assert(errmsg == NULL); + tt_str_op(answer, OP_EQ, dls_sample_1_str); + tor_free(question); + tor_free(answer); + errmsg = NULL; + + /* Case 3 - download status for default cert for 2nd auth id */ + memcpy(&auth_def_cert_download_status_2, &dls_sample_2, + sizeof(download_status_t)); + tor_asprintf(&question, "downloads/cert/fp/%s", auth_id_digest_2_str); + tt_assert(question != NULL); + getinfo_helper_downloads(&dummy, question, &answer, &errmsg); + tt_assert(answer != NULL); + tt_assert(errmsg == NULL); + tt_str_op(answer, OP_EQ, dls_sample_2_str); + tor_free(question); + tor_free(answer); + errmsg = NULL; + + /* Case 4 - list of signing key digests for 1st auth id */ + tor_asprintf(&question, "downloads/cert/fp/%s/sks", auth_id_digest_1_str); + tt_assert(question != NULL); + getinfo_helper_downloads(&dummy, question, &answer, &errmsg); + tt_assert(answer != NULL); + tt_assert(errmsg == NULL); + tt_str_op(answer, OP_EQ, auth_1_sk_digest_expected_list); + tor_free(question); + tor_free(answer); + errmsg = NULL; + + /* Case 5 - list of signing key digests for 2nd auth id */ + tor_asprintf(&question, "downloads/cert/fp/%s/sks", auth_id_digest_2_str); + tt_assert(question != NULL); + getinfo_helper_downloads(&dummy, question, &answer, &errmsg); + tt_assert(answer != NULL); + tt_assert(errmsg == NULL); + tt_str_op(answer, OP_EQ, auth_2_sk_digest_expected_list); + tor_free(question); + tor_free(answer); + errmsg = NULL; + + /* Case 6 - download status for 1st auth id, 1st sk */ + memcpy(&auth_1_sk_1_dls, &dls_sample_3, + sizeof(download_status_t)); + tor_asprintf(&question, "downloads/cert/fp/%s/%s", + auth_id_digest_1_str, auth_1_sk_1_str); + tt_assert(question != NULL); + getinfo_helper_downloads(&dummy, question, &answer, &errmsg); + tt_assert(answer != NULL); + tt_assert(errmsg == NULL); + tt_str_op(answer, OP_EQ, dls_sample_3_str); + tor_free(question); + tor_free(answer); + errmsg = NULL; + + /* Case 7 - download status for 1st auth id, 2nd sk */ + memcpy(&auth_1_sk_2_dls, &dls_sample_4, + sizeof(download_status_t)); + tor_asprintf(&question, "downloads/cert/fp/%s/%s", + auth_id_digest_1_str, auth_1_sk_2_str); + tt_assert(question != NULL); + getinfo_helper_downloads(&dummy, question, &answer, &errmsg); + tt_assert(answer != NULL); + tt_assert(errmsg == NULL); + tt_str_op(answer, OP_EQ, dls_sample_4_str); + tor_free(question); + tor_free(answer); + errmsg = NULL; + + /* Case 8 - download status for 2nd auth id, 1st sk */ + memcpy(&auth_2_sk_1_dls, &dls_sample_5, + sizeof(download_status_t)); + tor_asprintf(&question, "downloads/cert/fp/%s/%s", + auth_id_digest_2_str, auth_2_sk_1_str); + tt_assert(question != NULL); + getinfo_helper_downloads(&dummy, question, &answer, &errmsg); + tt_assert(answer != NULL); + tt_assert(errmsg == NULL); + tt_str_op(answer, OP_EQ, dls_sample_5_str); + tor_free(question); + tor_free(answer); + errmsg = NULL; + + /* Case 9 - download status for 2nd auth id, 2nd sk */ + memcpy(&auth_2_sk_2_dls, &dls_sample_6, + sizeof(download_status_t)); + tor_asprintf(&question, "downloads/cert/fp/%s/%s", + auth_id_digest_2_str, auth_2_sk_2_str); + tt_assert(question != NULL); + getinfo_helper_downloads(&dummy, question, &answer, &errmsg); + tt_assert(answer != NULL); + tt_assert(errmsg == NULL); + tt_str_op(answer, OP_EQ, dls_sample_6_str); + tor_free(question); + tor_free(answer); + errmsg = NULL; + + /* Now check the error cases */ + + /* Case 1 - query is garbage after downloads/cert/ part */ + getinfo_helper_downloads(&dummy, "downloads/cert/blahdeblah", + &answer, &errmsg); + tt_assert(answer == NULL); + tt_assert(errmsg != NULL); + tt_str_op(errmsg, OP_EQ, "Unknown certificate download status query"); + errmsg = NULL; + + /* + * Case 2 - looks like downloads/cert/fp/<fp>, but <fp> isn't even + * the right length for a digest. + */ + getinfo_helper_downloads(&dummy, "downloads/cert/fp/2B1D36D32B2942406", + &answer, &errmsg); + tt_assert(answer == NULL); + tt_assert(errmsg != NULL); + tt_str_op(errmsg, OP_EQ, "That didn't look like a digest"); + errmsg = NULL; + + /* + * Case 3 - looks like downloads/cert/fp/<fp>, and <fp> is digest-sized, + * but not parseable as one. + */ + getinfo_helper_downloads(&dummy, + "downloads/cert/fp/82F52AF55D250115FE44D3GC81D49643241D56A1", + &answer, &errmsg); + tt_assert(answer == NULL); + tt_assert(errmsg != NULL); + tt_str_op(errmsg, OP_EQ, "That didn't look like a digest"); + errmsg = NULL; + + /* + * Case 4 - downloads/cert/fp/<fp>, and <fp> is not a known authority + * identity digest + */ + getinfo_helper_downloads(&dummy, + "downloads/cert/fp/AC4F23B5745BDD2A77997B85B1FD85D05C2E0F61", + &answer, &errmsg); + tt_assert(answer == NULL); + tt_assert(errmsg != NULL); + tt_str_op(errmsg, OP_EQ, + "Failed to get download status for this authority identity digest"); + errmsg = NULL; + + /* + * Case 5 - looks like downloads/cert/fp/<fp>/<anything>, but <fp> doesn't + * parse as a sensible digest. + */ + getinfo_helper_downloads(&dummy, + "downloads/cert/fp/82F52AF55D250115FE44D3GC81D49643241D56A1/blah", + &answer, &errmsg); + tt_assert(answer == NULL); + tt_assert(errmsg != NULL); + tt_str_op(errmsg, OP_EQ, "That didn't look like an identity digest"); + errmsg = NULL; + + /* + * Case 6 - looks like downloads/cert/fp/<fp>/<anything>, but <fp> doesn't + * parse as a sensible digest. + */ + getinfo_helper_downloads(&dummy, + "downloads/cert/fp/82F52AF55D25/blah", + &answer, &errmsg); + tt_assert(answer == NULL); + tt_assert(errmsg != NULL); + tt_str_op(errmsg, OP_EQ, "That didn't look like an identity digest"); + errmsg = NULL; + + /* + * Case 7 - downloads/cert/fp/<fp>/sks, and <fp> is not a known authority + * digest. + */ + getinfo_helper_downloads(&dummy, + "downloads/cert/fp/AC4F23B5745BDD2A77997B85B1FD85D05C2E0F61/sks", + &answer, &errmsg); + tt_assert(answer == NULL); + tt_assert(errmsg != NULL); + tt_str_op(errmsg, OP_EQ, + "Failed to get list of signing key digests for this authority " + "identity digest"); + errmsg = NULL; + + /* + * Case 8 - looks like downloads/cert/fp/<fp>/<sk>, but <sk> doesn't + * parse as a signing key digest. + */ + getinfo_helper_downloads(&dummy, + "downloads/cert/fp/AC4F23B5745BDD2A77997B85B1FD85D05C2E0F61/" + "82F52AF55D250115FE44D3GC81D49643241D56A1", + &answer, &errmsg); + tt_assert(answer == NULL); + tt_assert(errmsg != NULL); + tt_str_op(errmsg, OP_EQ, "That didn't look like a signing key digest"); + errmsg = NULL; + + /* + * Case 9 - looks like downloads/cert/fp/<fp>/<sk>, but <sk> doesn't + * parse as a signing key digest. + */ + getinfo_helper_downloads(&dummy, + "downloads/cert/fp/AC4F23B5745BDD2A77997B85B1FD85D05C2E0F61/" + "82F52AF55D250115FE44D", + &answer, &errmsg); + tt_assert(answer == NULL); + tt_assert(errmsg != NULL); + tt_str_op(errmsg, OP_EQ, "That didn't look like a signing key digest"); + errmsg = NULL; + + /* + * Case 10 - downloads/cert/fp/<fp>/<sk>, but <fp> isn't a known + * authority identity digest. + */ + getinfo_helper_downloads(&dummy, + "downloads/cert/fp/C6B05DF332F74DB9A13498EE3BBC7AA2F69FCB45/" + "3A214FC21AE25B012C2ECCB5F4EC8A3602D0545D", + &answer, &errmsg); + tt_assert(answer == NULL); + tt_assert(errmsg != NULL); + tt_str_op(errmsg, OP_EQ, + "Failed to get download status for this identity/" + "signing key digest pair"); + errmsg = NULL; + + /* + * Case 11 - downloads/cert/fp/<fp>/<sk>, but <sk> isn't a known + * signing key digest. + */ + getinfo_helper_downloads(&dummy, + "downloads/cert/fp/63CDD326DFEF0CA020BDD3FEB45A3286FE13A061/" + "3A214FC21AE25B012C2ECCB5F4EC8A3602D0545D", + &answer, &errmsg); + tt_assert(answer == NULL); + tt_assert(errmsg != NULL); + tt_str_op(errmsg, OP_EQ, + "Failed to get download status for this identity/" + "signing key digest pair"); + errmsg = NULL; + + /* + * Case 12 - downloads/cert/fp/<fp>/<sk>, but <sk> is on the list for + * a different authority identity digest. + */ + getinfo_helper_downloads(&dummy, + "downloads/cert/fp/63CDD326DFEF0CA020BDD3FEB45A3286FE13A061/" + "9451B8F1B10952384EB58B5F230C0BB701626C9B", + &answer, &errmsg); + tt_assert(answer == NULL); + tt_assert(errmsg != NULL); + tt_str_op(errmsg, OP_EQ, + "Failed to get download status for this identity/" + "signing key digest pair"); + errmsg = NULL; + + done: + clear_cert_mocks(); + tor_free(answer); + + return; +} + +static void +test_download_status_desc(void *arg) +{ + /* We just need one of these to pass, it doesn't matter what's in it */ + control_connection_t dummy; + /* Get results out */ + char *question = NULL; + char *answer = NULL; + const char *errmsg = NULL; + + (void)arg; + + setup_desc_mocks(); + + /* + * Check returning serialized dlstatuses and digest lists, and implicitly + * also test download_status_to_string() and digest_list_to_string(). + */ + + /* Case 1 - list of router descriptor digests */ + getinfo_helper_downloads(&dummy, + "downloads/desc/descs", + &answer, &errmsg); + tt_assert(answer != NULL); + tt_assert(errmsg == NULL); + tt_str_op(answer, OP_EQ, descbr_expected_list); + tor_free(answer); + errmsg = NULL; + + /* Case 2 - get download status for router descriptor 1 */ + memcpy(&descbr_digest_1_dl, &dls_sample_1, + sizeof(download_status_t)); + tor_asprintf(&question, "downloads/desc/%s", descbr_digest_1_str); + tt_assert(question != NULL); + getinfo_helper_downloads(&dummy, question, &answer, &errmsg); + tt_assert(answer != NULL); + tt_assert(errmsg == NULL); + tt_str_op(answer, OP_EQ, dls_sample_1_str); + tor_free(question); + tor_free(answer); + errmsg = NULL; + + /* Case 3 - get download status for router descriptor 1 */ + memcpy(&descbr_digest_2_dl, &dls_sample_2, + sizeof(download_status_t)); + tor_asprintf(&question, "downloads/desc/%s", descbr_digest_2_str); + tt_assert(question != NULL); + getinfo_helper_downloads(&dummy, question, &answer, &errmsg); + tt_assert(answer != NULL); + tt_assert(errmsg == NULL); + tt_str_op(answer, OP_EQ, dls_sample_2_str); + tor_free(question); + tor_free(answer); + errmsg = NULL; + + /* Now check the error cases */ + + /* Case 1 - non-digest-length garbage after downloads/desc */ + getinfo_helper_downloads(&dummy, "downloads/desc/blahdeblah", + &answer, &errmsg); + tt_assert(answer == NULL); + tt_assert(errmsg != NULL); + tt_str_op(errmsg, OP_EQ, "Unknown router descriptor download status query"); + errmsg = NULL; + + /* Case 2 - nonparseable digest-shaped thing */ + getinfo_helper_downloads( + &dummy, + "downloads/desc/774EC52FD9A5B80A6FACZE536616E8022E3470AG", + &answer, &errmsg); + tt_assert(answer == NULL); + tt_assert(errmsg != NULL); + tt_str_op(errmsg, OP_EQ, "That didn't look like a digest"); + errmsg = NULL; + + /* Case 3 - digest we have no descriptor for */ + getinfo_helper_downloads( + &dummy, + "downloads/desc/B05B46135B0B2C04EBE1DD6A6AE4B12D7CD2226A", + &answer, &errmsg); + tt_assert(answer == NULL); + tt_assert(errmsg != NULL); + tt_str_op(errmsg, OP_EQ, "No such descriptor digest found"); + errmsg = NULL; + + /* Case 4 - microdescs only */ + disable_descbr = 1; + getinfo_helper_downloads(&dummy, + "downloads/desc/descs", + &answer, &errmsg); + tt_assert(answer == NULL); + tt_assert(errmsg != NULL); + tt_str_op(errmsg, OP_EQ, + "We don't seem to have a networkstatus-flavored consensus"); + errmsg = NULL; + disable_descbr = 0; + + done: + clear_desc_mocks(); + tor_free(answer); + + return; +} + +static void +test_download_status_bridge(void *arg) +{ + /* We just need one of these to pass, it doesn't matter what's in it */ + control_connection_t dummy; + /* Get results out */ + char *question = NULL; + char *answer = NULL; + const char *errmsg = NULL; + + (void)arg; + + setup_bridge_mocks(); + + /* + * Check returning serialized dlstatuses and digest lists, and implicitly + * also test download_status_to_string() and digest_list_to_string(). + */ + + /* Case 1 - list of bridge identity digests */ + getinfo_helper_downloads(&dummy, + "downloads/bridge/bridges", + &answer, &errmsg); + tt_assert(answer != NULL); + tt_assert(errmsg == NULL); + tt_str_op(answer, OP_EQ, descbr_expected_list); + tor_free(answer); + errmsg = NULL; + + /* Case 2 - get download status for bridge descriptor 1 */ + memcpy(&descbr_digest_1_dl, &dls_sample_3, + sizeof(download_status_t)); + tor_asprintf(&question, "downloads/bridge/%s", descbr_digest_1_str); + tt_assert(question != NULL); + getinfo_helper_downloads(&dummy, question, &answer, &errmsg); + tt_assert(answer != NULL); + tt_assert(errmsg == NULL); + tt_str_op(answer, OP_EQ, dls_sample_3_str); + tor_free(question); + tor_free(answer); + errmsg = NULL; + + /* Case 3 - get download status for router descriptor 1 */ + memcpy(&descbr_digest_2_dl, &dls_sample_4, + sizeof(download_status_t)); + tor_asprintf(&question, "downloads/bridge/%s", descbr_digest_2_str); + tt_assert(question != NULL); + getinfo_helper_downloads(&dummy, question, &answer, &errmsg); + tt_assert(answer != NULL); + tt_assert(errmsg == NULL); + tt_str_op(answer, OP_EQ, dls_sample_4_str); + tor_free(question); + tor_free(answer); + errmsg = NULL; + + /* Now check the error cases */ + + /* Case 1 - non-digest-length garbage after downloads/bridge */ + getinfo_helper_downloads(&dummy, "downloads/bridge/blahdeblah", + &answer, &errmsg); + tt_assert(answer == NULL); + tt_assert(errmsg != NULL); + tt_str_op(errmsg, OP_EQ, "Unknown bridge descriptor download status query"); + errmsg = NULL; + + /* Case 2 - nonparseable digest-shaped thing */ + getinfo_helper_downloads( + &dummy, + "downloads/bridge/774EC52FD9A5B80A6FACZE536616E8022E3470AG", + &answer, &errmsg); + tt_assert(answer == NULL); + tt_assert(errmsg != NULL); + tt_str_op(errmsg, OP_EQ, "That didn't look like a digest"); + errmsg = NULL; + + /* Case 3 - digest we have no descriptor for */ + getinfo_helper_downloads( + &dummy, + "downloads/bridge/B05B46135B0B2C04EBE1DD6A6AE4B12D7CD2226A", + &answer, &errmsg); + tt_assert(answer == NULL); + tt_assert(errmsg != NULL); + tt_str_op(errmsg, OP_EQ, "No such bridge identity digest found"); + errmsg = NULL; + + /* Case 4 - bridges disabled */ + disable_descbr = 1; + getinfo_helper_downloads(&dummy, + "downloads/bridge/bridges", + &answer, &errmsg); + tt_assert(answer == NULL); + tt_assert(errmsg != NULL); + tt_str_op(errmsg, OP_EQ, "We don't seem to be using bridges"); + errmsg = NULL; + disable_descbr = 0; + + done: + clear_bridge_mocks(); + tor_free(answer); + + return; +} + struct testcase_t controller_tests[] = { { "add_onion_helper_keyarg", test_add_onion_helper_keyarg, 0, NULL, NULL }, { "rend_service_parse_port_config", test_rend_service_parse_port_config, 0, NULL, NULL }, + { "add_onion_helper_clientauth", test_add_onion_helper_clientauth, 0, NULL, + NULL }, + { "download_status_consensus", test_download_status_consensus, 0, NULL, + NULL }, + { "download_status_cert", test_download_status_cert, 0, NULL, + NULL }, + { "download_status_desc", test_download_status_desc, 0, NULL, NULL }, + { "download_status_bridge", test_download_status_bridge, 0, NULL, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c index 6a95e92733..ba2fb86246 100644 --- a/src/test/test_crypto.c +++ b/src/test/test_crypto.c @@ -18,15 +18,12 @@ #include <openssl/evp.h> #include <openssl/rand.h> -extern const char AUTHORITY_SIGNKEY_3[]; -extern const char AUTHORITY_SIGNKEY_A_DIGEST[]; -extern const char AUTHORITY_SIGNKEY_A_DIGEST256[]; - /** Run unit tests for Diffie-Hellman functionality. */ static void test_crypto_dh(void *arg) { crypto_dh_t *dh1 = crypto_dh_new(DH_TYPE_CIRCUIT); + crypto_dh_t *dh1_dup = NULL; crypto_dh_t *dh2 = crypto_dh_new(DH_TYPE_CIRCUIT); char p1[DH_BYTES]; char p2[DH_BYTES]; @@ -41,6 +38,9 @@ test_crypto_dh(void *arg) memset(p1, 0, DH_BYTES); memset(p2, 0, DH_BYTES); tt_mem_op(p1,OP_EQ, p2, DH_BYTES); + + tt_int_op(-1, OP_EQ, crypto_dh_get_public(dh1, p1, 6)); /* too short */ + tt_assert(! crypto_dh_get_public(dh1, p1, DH_BYTES)); tt_mem_op(p1,OP_NE, p2, DH_BYTES); tt_assert(! crypto_dh_get_public(dh2, p2, DH_BYTES)); @@ -54,15 +54,119 @@ test_crypto_dh(void *arg) tt_int_op(s1len,OP_EQ, s2len); tt_mem_op(s1,OP_EQ, s2, s1len); + /* test dh_dup; make sure it works the same. */ + dh1_dup = crypto_dh_dup(dh1); + s1len = crypto_dh_compute_secret(LOG_WARN, dh1_dup, p2, DH_BYTES, s1, 50); + tt_mem_op(s1,OP_EQ, s2, s1len); + { - /* XXXX Now fabricate some bad values and make sure they get caught, - * Check 0, 1, N-1, >= N, etc. - */ + /* Now fabricate some bad values and make sure they get caught. */ + + /* 1 and 0 should both fail. */ + s1len = crypto_dh_compute_secret(LOG_WARN, dh1, "\x01", 1, s1, 50); + tt_int_op(-1, OP_EQ, s1len); + + s1len = crypto_dh_compute_secret(LOG_WARN, dh1, "\x00", 1, s1, 50); + tt_int_op(-1, OP_EQ, s1len); + + memset(p1, 0, DH_BYTES); /* 0 with padding. */ + s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH_BYTES, s1, 50); + tt_int_op(-1, OP_EQ, s1len); + + p1[DH_BYTES-1] = 1; /* 1 with padding*/ + s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH_BYTES, s1, 50); + tt_int_op(-1, OP_EQ, s1len); + + /* 2 is okay, though weird. */ + s1len = crypto_dh_compute_secret(LOG_WARN, dh1, "\x02", 1, s1, 50); + tt_int_op(50, OP_EQ, s1len); + + const char P[] = + "FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD129024E08" + "8A67CC74020BBEA63B139B22514A08798E3404DDEF9519B3CD3A431B" + "302B0A6DF25F14374FE1356D6D51C245E485B576625E7EC6F44C42E9" + "A637ED6B0BFF5CB6F406B7EDEE386BFB5A899FA5AE9F24117C4B1FE6" + "49286651ECE65381FFFFFFFFFFFFFFFF"; + + /* p-1, p, and so on are not okay. */ + base16_decode(p1, sizeof(p1), P, strlen(P)); + + s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH_BYTES, s1, 50); + tt_int_op(-1, OP_EQ, s1len); + + p1[DH_BYTES-1] = 0xFE; /* p-1 */ + s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH_BYTES, s1, 50); + tt_int_op(-1, OP_EQ, s1len); + + p1[DH_BYTES-1] = 0xFD; /* p-2 works fine */ + s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH_BYTES, s1, 50); + tt_int_op(50, OP_EQ, s1len); + + const char P_plus_one[] = + "FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD129024E08" + "8A67CC74020BBEA63B139B22514A08798E3404DDEF9519B3CD3A431B" + "302B0A6DF25F14374FE1356D6D51C245E485B576625E7EC6F44C42E9" + "A637ED6B0BFF5CB6F406B7EDEE386BFB5A899FA5AE9F24117C4B1FE6" + "49286651ECE653820000000000000000"; + + base16_decode(p1, sizeof(p1), P_plus_one, strlen(P_plus_one)); + + s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH_BYTES, s1, 50); + tt_int_op(-1, OP_EQ, s1len); + + p1[DH_BYTES-1] = 0x01; /* p+2 */ + s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH_BYTES, s1, 50); + tt_int_op(-1, OP_EQ, s1len); + + p1[DH_BYTES-1] = 0xff; /* p+256 */ + s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH_BYTES, s1, 50); + tt_int_op(-1, OP_EQ, s1len); + + memset(p1, 0xff, DH_BYTES), /* 2^1024-1 */ + s1len = crypto_dh_compute_secret(LOG_WARN, dh1, p1, DH_BYTES, s1, 50); + tt_int_op(-1, OP_EQ, s1len); + } + + { + /* provoke an error in the openssl DH_compute_key function; make sure we + * survive. */ + tt_assert(! crypto_dh_get_public(dh1, p1, DH_BYTES)); + + crypto_dh_free(dh2); + dh2= crypto_dh_new(DH_TYPE_CIRCUIT); /* no private key set */ + s1len = crypto_dh_compute_secret(LOG_WARN, dh2, + p1, DH_BYTES, + s1, 50); + tt_int_op(s1len, OP_EQ, -1); } done: crypto_dh_free(dh1); crypto_dh_free(dh2); + crypto_dh_free(dh1_dup); +} + +static void +test_crypto_openssl_version(void *arg) +{ + (void)arg; + const char *version = crypto_openssl_get_version_str(); + const char *h_version = crypto_openssl_get_header_version_str(); + tt_assert(version); + tt_assert(h_version); + tt_assert(!strcmpstart(version, h_version)); /* "-fips" suffix, etc */ + tt_assert(!strstr(version, "OpenSSL")); + int a=-1,b=-1,c=-1; + if (!strcmpstart(version, "LibreSSL") || !strcmpstart(version, "BoringSSL")) + return; + int r = tor_sscanf(version, "%d.%d.%d", &a,&b,&c); + tt_int_op(r, OP_EQ, 3); + tt_int_op(a, OP_GE, 0); + tt_int_op(b, OP_GE, 0); + tt_int_op(c, OP_GE, 0); + + done: + ; } /** Run unit tests for our random number generation function and its wrappers. @@ -73,6 +177,7 @@ test_crypto_rng(void *arg) int i, j, allok; char data1[100], data2[100]; double d; + char *h=NULL; /* Try out RNG. */ (void)arg; @@ -104,9 +209,16 @@ test_crypto_rng(void *arg) allok = 0; tor_free(host); } + + /* Make sure crypto_random_hostname clips its inputs properly. */ + h = crypto_random_hostname(20000, 9000, "www.", ".onion"); + tt_assert(! strcmpstart(h,"www.")); + tt_assert(! strcmpend(h,".onion")); + tt_int_op(63+4+6, OP_EQ, strlen(h)); + tt_assert(allok); done: - ; + tor_free(h); } static void @@ -125,14 +237,100 @@ test_crypto_rng_range(void *arg) if (x == 8) got_largest = 1; } - /* These fail with probability 1/10^603. */ tt_assert(got_smallest); tt_assert(got_largest); + + got_smallest = got_largest = 0; + const uint64_t ten_billion = 10 * ((uint64_t)1000000000000); + for (i = 0; i < 1000; ++i) { + uint64_t x = crypto_rand_uint64_range(ten_billion, ten_billion+10); + tt_u64_op(x, OP_GE, ten_billion); + tt_u64_op(x, OP_LT, ten_billion+10); + if (x == ten_billion) + got_smallest = 1; + if (x == ten_billion+9) + got_largest = 1; + } + + tt_assert(got_smallest); + tt_assert(got_largest); + + const time_t now = time(NULL); + for (i = 0; i < 2000; ++i) { + time_t x = crypto_rand_time_range(now, now+60); + tt_i64_op(x, OP_GE, now); + tt_i64_op(x, OP_LT, now+60); + if (x == now) + got_smallest = 1; + if (x == now+59) + got_largest = 1; + } + + tt_assert(got_smallest); + tt_assert(got_largest); done: ; } +static void +test_crypto_rng_strongest(void *arg) +{ + const char *how = arg; + int broken = 0; + + if (how == NULL) { + ; + } else if (!strcmp(how, "nosyscall")) { + break_strongest_rng_syscall = 1; + } else if (!strcmp(how, "nofallback")) { + break_strongest_rng_fallback = 1; + } else if (!strcmp(how, "broken")) { + broken = break_strongest_rng_syscall = break_strongest_rng_fallback = 1; + } + +#define N 128 + uint8_t combine_and[N]; + uint8_t combine_or[N]; + int i, j; + + memset(combine_and, 0xff, N); + memset(combine_or, 0, N); + + for (i = 0; i < 100; ++i) { /* 2^-100 chances just don't happen. */ + uint8_t output[N]; + memset(output, 0, N); + if (how == NULL) { + /* this one can't fail. */ + crypto_strongest_rand(output, sizeof(output)); + } else { + int r = crypto_strongest_rand_raw(output, sizeof(output)); + if (r == -1) { + if (broken) { + goto done; /* we're fine. */ + } + /* This function is allowed to break, but only if it always breaks. */ + tt_int_op(i, OP_EQ, 0); + tt_skip(); + } else { + tt_assert(! broken); + } + } + for (j = 0; j < N; ++j) { + combine_and[j] &= output[j]; + combine_or[j] |= output[j]; + } + } + + for (j = 0; j < N; ++j) { + tt_int_op(combine_and[j], OP_EQ, 0); + tt_int_op(combine_or[j], OP_EQ, 0xff); + } + done: + ; +#undef N +} + /* Test for rectifying openssl RAND engine. */ static void test_crypto_rng_engine(void *arg) @@ -312,6 +510,43 @@ test_crypto_aes(void *arg) tor_free(data3); } +static void +test_crypto_aes_ctr_testvec(void *arg) +{ + (void)arg; + char *mem_op_hex_tmp=NULL; + + /* from NIST SP800-38a, section F.5 */ + const char key16[] = "2b7e151628aed2a6abf7158809cf4f3c"; + const char ctr16[] = "f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff"; + const char plaintext16[] = + "6bc1bee22e409f96e93d7e117393172a" + "ae2d8a571e03ac9c9eb76fac45af8e51" + "30c81c46a35ce411e5fbc1191a0a52ef" + "f69f2445df4f9b17ad2b417be66c3710"; + const char ciphertext16[] = + "874d6191b620e3261bef6864990db6ce" + "9806f66b7970fdff8617187bb9fffdff" + "5ae4df3edbd5d35e5b4f09020db03eab" + "1e031dda2fbe03d1792170a0f3009cee"; + + char key[16]; + char iv[16]; + char plaintext[16*4]; + base16_decode(key, sizeof(key), key16, strlen(key16)); + base16_decode(iv, sizeof(iv), ctr16, strlen(ctr16)); + base16_decode(plaintext, sizeof(plaintext), + plaintext16, strlen(plaintext16)); + + crypto_cipher_t *c = crypto_cipher_new_with_iv(key, iv); + crypto_cipher_crypt_inplace(c, plaintext, sizeof(plaintext)); + test_memeq_hex(plaintext, ciphertext16); + + done: + tor_free(mem_op_hex_tmp); + crypto_cipher_free(c); +} + /** Run unit tests for our SHA-1 functionality */ static void test_crypto_sha(void *arg) @@ -1084,6 +1319,29 @@ test_crypto_pk_base64(void *arg) tor_free(encoded); } +#ifdef HAVE_TRUNCATE +#define do_truncate truncate +#else +static int +do_truncate(const char *fname, size_t len) +{ + struct stat st; + char *bytes; + + bytes = read_file_to_str(fname, RFTS_BIN, &st); + if (!bytes) + return -1; + /* This cast isn't so great, but it should be safe given the actual files + * and lengths we're using. */ + if (st.st_size < (off_t)len) + len = MIN(len, (size_t)st.st_size); + + int r = write_bytes_to_file(fname, bytes, len, 1); + tor_free(bytes); + return r; +} +#endif + /** Sanity check for crypto pk digests */ static void test_crypto_digests(void *arg) @@ -1114,6 +1372,33 @@ test_crypto_digests(void *arg) crypto_pk_free(k); } +static void +test_crypto_digest_names(void *arg) +{ + static const struct { + int a; const char *n; + } names[] = { + { DIGEST_SHA1, "sha1" }, + { DIGEST_SHA256, "sha256" }, + { DIGEST_SHA512, "sha512" }, + { DIGEST_SHA3_256, "sha3-256" }, + { DIGEST_SHA3_512, "sha3-512" }, + { -1, NULL } + }; + (void)arg; + + int i; + for (i = 0; names[i].n; ++i) { + tt_str_op(names[i].n, OP_EQ,crypto_digest_algorithm_get_name(names[i].a)); + tt_int_op(names[i].a, + OP_EQ,crypto_digest_algorithm_parse_name(names[i].n)); + } + tt_int_op(-1, OP_EQ, + crypto_digest_algorithm_parse_name("TimeCubeHash-4444")); + done: + ; +} + #ifndef OPENSSL_1_1_API #define EVP_ENCODE_CTX_new() tor_malloc_zero(sizeof(EVP_ENCODE_CTX)) #define EVP_ENCODE_CTX_free(ctx) tor_free(ctx) @@ -1236,7 +1521,7 @@ test_crypto_formats(void *arg) strlcpy(data1, "f0d678affc000100", 1024); i = base16_decode(data2, 8, data1, 16); - tt_int_op(i,OP_EQ, 0); + tt_int_op(i,OP_EQ, 8); tt_mem_op(data2,OP_EQ, "\xf0\xd6\x78\xaf\xfc\x00\x01\x00",8); /* now try some failing base16 decodes */ @@ -1507,13 +1792,98 @@ test_crypto_hkdf_sha256(void *arg) "b206fa34e5bc78d063fc291501beec53b36e5a0e434561200c" "5f8bd13e0f88b3459600b4dc21d69363e2895321c06184879d" "94b18f078411be70b767c7fc40679a9440a0c95ea83a23efbf"); - done: tor_free(mem_op_hex_tmp); #undef EXPAND } static void +test_crypto_hkdf_sha256_testvecs(void *arg) +{ + (void) arg; + /* Test vectors from RFC5869, sections A.1 through A.3 */ + const struct { + const char *ikm16, *salt16, *info16; + int L; + const char *okm16; + } vecs[] = { + { /* from A.1 */ + "0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b", + "000102030405060708090a0b0c", + "f0f1f2f3f4f5f6f7f8f9", + 42, + "3cb25f25faacd57a90434f64d0362f2a2d2d0a90cf1a5a4c5db02d56ecc4c5bf" + "34007208d5b887185865" + }, + { /* from A.2 */ + "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f" + "202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f" + "404142434445464748494a4b4c4d4e4f", + "606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f" + "808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9f" + "a0a1a2a3a4a5a6a7a8a9aaabacadaeaf", + "b0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecf" + "d0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeef" + "f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff", + 82, + "b11e398dc80327a1c8e7f78c596a49344f012eda2d4efad8a050cc4c19afa97c" + "59045a99cac7827271cb41c65e590e09da3275600c2f09b8367793a9aca3db71" + "cc30c58179ec3e87c14c01d5c1f3434f1d87" + }, + { /* from A.3 */ + "0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b", + "", + "", + 42, + "8da4e775a563c18f715f802a063c5a31b8a11f5c5ee1879ec3454e5f3c738d2d" + "9d201395faa4b61a96c8", + }, + { NULL, NULL, NULL, -1, NULL } + }; + + int i; + char *ikm = NULL; + char *salt = NULL; + char *info = NULL; + char *okm = NULL; + char *mem_op_hex_tmp = NULL; + + for (i = 0; vecs[i].ikm16; ++i) { + size_t ikm_len = strlen(vecs[i].ikm16)/2; + size_t salt_len = strlen(vecs[i].salt16)/2; + size_t info_len = strlen(vecs[i].info16)/2; + size_t okm_len = vecs[i].L; + + ikm = tor_malloc(ikm_len); + salt = tor_malloc(salt_len); + info = tor_malloc(info_len); + okm = tor_malloc(okm_len); + + base16_decode(ikm, ikm_len, vecs[i].ikm16, strlen(vecs[i].ikm16)); + base16_decode(salt, salt_len, vecs[i].salt16, strlen(vecs[i].salt16)); + base16_decode(info, info_len, vecs[i].info16, strlen(vecs[i].info16)); + + int r = crypto_expand_key_material_rfc5869_sha256( + (const uint8_t*)ikm, ikm_len, + (const uint8_t*)salt, salt_len, + (const uint8_t*)info, info_len, + (uint8_t*)okm, okm_len); + tt_int_op(r, OP_EQ, 0); + test_memeq_hex(okm, vecs[i].okm16); + tor_free(ikm); + tor_free(salt); + tor_free(info); + tor_free(okm); + } + done: + tor_free(ikm); + tor_free(salt); + tor_free(info); + tor_free(okm); + tor_free(mem_op_hex_tmp); +} + +static void test_crypto_curve25519_impl(void *arg) { /* adapted from curve25519_donna, which adapted it from test-curve25519 @@ -1605,6 +1975,47 @@ test_crypto_curve25519_basepoint(void *arg) } static void +test_crypto_curve25519_testvec(void *arg) +{ + (void)arg; + char *mem_op_hex_tmp = NULL; + + /* From RFC 7748, section 6.1 */ + /* Alice's private key, a: */ + const char a16[] = + "77076d0a7318a57d3c16c17251b26645df4c2f87ebc0992ab177fba51db92c2a"; + /* Alice's public key, X25519(a, 9): */ + const char a_pub16[] = + "8520f0098930a754748b7ddcb43ef75a0dbf3a0d26381af4eba4a98eaa9b4e6a"; + /* Bob's private key, b: */ + const char b16[] = + "5dab087e624a8a4b79e17f8b83800ee66f3bb1292618b6fd1c2f8b27ff88e0eb"; + /* Bob's public key, X25519(b, 9): */ + const char b_pub16[] = + "de9edb7d7b7dc1b4d35b61c2ece435373f8343c85b78674dadfc7e146f882b4f"; + /* Their shared secret, K: */ + const char k16[] = + "4a5d9d5ba4ce2de1728e3bf480350f25e07e21c947d19e3376f09b3c1e161742"; + + uint8_t a[32], b[32], a_pub[32], b_pub[32], k1[32], k2[32]; + base16_decode((char*)a, sizeof(a), a16, strlen(a16)); + base16_decode((char*)b, sizeof(b), b16, strlen(b16)); + curve25519_basepoint_impl(a_pub, a); + curve25519_basepoint_impl(b_pub, b); + curve25519_impl(k1, a, b_pub); + curve25519_impl(k2, b, a_pub); + + test_memeq_hex(a, a16); + test_memeq_hex(b, b16); + test_memeq_hex(a_pub, a_pub16); + test_memeq_hex(b_pub, b_pub16); + test_memeq_hex(k1, k16); + test_memeq_hex(k2, k16); + done: + tor_free(mem_op_hex_tmp); +} + +static void test_crypto_curve25519_wrappers(void *arg) { curve25519_public_key_t pubkey1, pubkey2; @@ -1896,7 +2307,67 @@ test_crypto_ed25519_test_vectors(void *arg) "1fbc1e08682f2cc0c92efe8f4985dec61dcbd54d4b94a22547d24451271c8b00", "0a688e79be24f866286d4646b5d81c" }, - + /* These come from draft-irtf-cfrg-eddsa-05 section 7.1 */ + { + "9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60", + "d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a", + "e5564300c360ac729086e2cc806e828a84877f1eb8e5d974d873e06522490155" + "5fb8821590a33bacc61e39701cf9b46bd25bf5f0595bbe24655141438e7a100b", + "" + }, + { + "4ccd089b28ff96da9db6c346ec114e0f5b8a319f35aba624da8cf6ed4fb8a6fb", + "3d4017c3e843895a92b70aa74d1b7ebc9c982ccf2ec4968cc0cd55f12af4660c", + "92a009a9f0d4cab8720e820b5f642540a2b27b5416503f8fb3762223ebdb69da" + "085ac1e43e15996e458f3613d0f11d8c387b2eaeb4302aeeb00d291612bb0c00", + "72" + }, + { + "f5e5767cf153319517630f226876b86c8160cc583bc013744c6bf255f5cc0ee5", + "278117fc144c72340f67d0f2316e8386ceffbf2b2428c9c51fef7c597f1d426e", + "0aab4c900501b3e24d7cdf4663326a3a87df5e4843b2cbdb67cbf6e460fec350" + "aa5371b1508f9f4528ecea23c436d94b5e8fcd4f681e30a6ac00a9704a188a03", + "08b8b2b733424243760fe426a4b54908632110a66c2f6591eabd3345e3e4eb98" + "fa6e264bf09efe12ee50f8f54e9f77b1e355f6c50544e23fb1433ddf73be84d8" + "79de7c0046dc4996d9e773f4bc9efe5738829adb26c81b37c93a1b270b20329d" + "658675fc6ea534e0810a4432826bf58c941efb65d57a338bbd2e26640f89ffbc" + "1a858efcb8550ee3a5e1998bd177e93a7363c344fe6b199ee5d02e82d522c4fe" + "ba15452f80288a821a579116ec6dad2b3b310da903401aa62100ab5d1a36553e" + "06203b33890cc9b832f79ef80560ccb9a39ce767967ed628c6ad573cb116dbef" + "efd75499da96bd68a8a97b928a8bbc103b6621fcde2beca1231d206be6cd9ec7" + "aff6f6c94fcd7204ed3455c68c83f4a41da4af2b74ef5c53f1d8ac70bdcb7ed1" + "85ce81bd84359d44254d95629e9855a94a7c1958d1f8ada5d0532ed8a5aa3fb2" + "d17ba70eb6248e594e1a2297acbbb39d502f1a8c6eb6f1ce22b3de1a1f40cc24" + "554119a831a9aad6079cad88425de6bde1a9187ebb6092cf67bf2b13fd65f270" + "88d78b7e883c8759d2c4f5c65adb7553878ad575f9fad878e80a0c9ba63bcbcc" + "2732e69485bbc9c90bfbd62481d9089beccf80cfe2df16a2cf65bd92dd597b07" + "07e0917af48bbb75fed413d238f5555a7a569d80c3414a8d0859dc65a46128ba" + "b27af87a71314f318c782b23ebfe808b82b0ce26401d2e22f04d83d1255dc51a" + "ddd3b75a2b1ae0784504df543af8969be3ea7082ff7fc9888c144da2af58429e" + "c96031dbcad3dad9af0dcbaaaf268cb8fcffead94f3c7ca495e056a9b47acdb7" + "51fb73e666c6c655ade8297297d07ad1ba5e43f1bca32301651339e22904cc8c" + "42f58c30c04aafdb038dda0847dd988dcda6f3bfd15c4b4c4525004aa06eeff8" + "ca61783aacec57fb3d1f92b0fe2fd1a85f6724517b65e614ad6808d6f6ee34df" + "f7310fdc82aebfd904b01e1dc54b2927094b2db68d6f903b68401adebf5a7e08" + "d78ff4ef5d63653a65040cf9bfd4aca7984a74d37145986780fc0b16ac451649" + "de6188a7dbdf191f64b5fc5e2ab47b57f7f7276cd419c17a3ca8e1b939ae49e4" + "88acba6b965610b5480109c8b17b80e1b7b750dfc7598d5d5011fd2dcc5600a3" + "2ef5b52a1ecc820e308aa342721aac0943bf6686b64b2579376504ccc493d97e" + "6aed3fb0f9cd71a43dd497f01f17c0e2cb3797aa2a2f256656168e6c496afc5f" + "b93246f6b1116398a346f1a641f3b041e989f7914f90cc2c7fff357876e506b5" + "0d334ba77c225bc307ba537152f3f1610e4eafe595f6d9d90d11faa933a15ef1" + "369546868a7f3a45a96768d40fd9d03412c091c6315cf4fde7cb68606937380d" + "b2eaaa707b4c4185c32eddcdd306705e4dc1ffc872eeee475a64dfac86aba41c" + "0618983f8741c5ef68d3a101e8a3b8cac60c905c15fc910840b94c00a0b9d0" + }, + { + "833fe62409237b9d62ec77587520911e9a759cec1d19755b7da901b96dca3d42", + "ec172b93ad5e563bf4932c70e1245034c35467ef2efd4d64ebf819683467e2bf", + "dc2a4459e7369633a52b1bf277839a00201009a3efbf3ecb69bea2186c26b589" + "09351fc9ac90b3ecfdfbc7c66431e0303dca179c138ac17ad9bef1177331a704", + "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a" + "2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f" + }, { NULL, NULL, NULL, NULL} }; @@ -2066,8 +2537,9 @@ test_crypto_ed25519_testvectors(void *arg) #define DECODE(p,s) base16_decode((char*)(p),sizeof(p),(s),strlen(s)) #define EQ(a,h) test_memeq_hex((const char*)(a), (h)) - tt_int_op(0, OP_EQ, DECODE(sk, ED25519_SECRET_KEYS[i])); - tt_int_op(0, OP_EQ, DECODE(blinding_param, ED25519_BLINDING_PARAMS[i])); + tt_int_op(sizeof(sk), OP_EQ, DECODE(sk, ED25519_SECRET_KEYS[i])); + tt_int_op(sizeof(blinding_param), OP_EQ, DECODE(blinding_param, + ED25519_BLINDING_PARAMS[i])); tt_int_op(0, OP_EQ, ed25519_secret_key_from_seed(&esk, sk)); EQ(esk.seckey, ED25519_EXPANDED_SECRET_KEYS[i]); @@ -2183,6 +2655,54 @@ test_crypto_ed25519_fuzz_donna(void *arg) } static void +test_crypto_ed25519_storage(void *arg) +{ + (void)arg; + ed25519_keypair_t *keypair = NULL; + ed25519_public_key_t pub; + ed25519_secret_key_t sec; + char *fname_1 = tor_strdup(get_fname("ed_seckey_1")); + char *fname_2 = tor_strdup(get_fname("ed_pubkey_2")); + char *contents = NULL; + char *tag = NULL; + + keypair = tor_malloc_zero(sizeof(ed25519_keypair_t)); + tt_int_op(0,OP_EQ,ed25519_keypair_generate(keypair, 0)); + tt_int_op(0,OP_EQ, + ed25519_seckey_write_to_file(&keypair->seckey, fname_1, "foo")); + tt_int_op(0,OP_EQ, + ed25519_pubkey_write_to_file(&keypair->pubkey, fname_2, "bar")); + + tt_int_op(-1, OP_EQ, ed25519_pubkey_read_from_file(&pub, &tag, fname_1)); + tt_ptr_op(tag, OP_EQ, NULL); + tt_int_op(-1, OP_EQ, ed25519_seckey_read_from_file(&sec, &tag, fname_2)); + tt_ptr_op(tag, OP_EQ, NULL); + + tt_int_op(0, OP_EQ, ed25519_pubkey_read_from_file(&pub, &tag, fname_2)); + tt_str_op(tag, OP_EQ, "bar"); + tor_free(tag); + tt_int_op(0, OP_EQ, ed25519_seckey_read_from_file(&sec, &tag, fname_1)); + tt_str_op(tag, OP_EQ, "foo"); + tor_free(tag); + + /* whitebox test: truncated keys. */ + tt_int_op(0, ==, do_truncate(fname_1, 40)); + tt_int_op(0, ==, do_truncate(fname_2, 40)); + tt_int_op(-1, OP_EQ, ed25519_pubkey_read_from_file(&pub, &tag, fname_2)); + tt_ptr_op(tag, OP_EQ, NULL); + tor_free(tag); + tt_int_op(-1, OP_EQ, ed25519_seckey_read_from_file(&sec, &tag, fname_1)); + tt_ptr_op(tag, OP_EQ, NULL); + + done: + tor_free(fname_1); + tor_free(fname_2); + tor_free(contents); + tor_free(tag); + ed25519_keypair_free(keypair); +} + +static void test_crypto_siphash(void *arg) { /* From the reference implementation, taking @@ -2398,13 +2918,23 @@ struct testcase_t crypto_tests[] = { CRYPTO_LEGACY(rng), { "rng_range", test_crypto_rng_range, 0, NULL, NULL }, { "rng_engine", test_crypto_rng_engine, TT_FORK, NULL, NULL }, + { "rng_strongest", test_crypto_rng_strongest, TT_FORK, NULL, NULL }, + { "rng_strongest_nosyscall", test_crypto_rng_strongest, TT_FORK, + &passthrough_setup, (void*)"nosyscall" }, + { "rng_strongest_nofallback", test_crypto_rng_strongest, TT_FORK, + &passthrough_setup, (void*)"nofallback" }, + { "rng_strongest_broken", test_crypto_rng_strongest, TT_FORK, + &passthrough_setup, (void*)"broken" }, + { "openssl_version", test_crypto_openssl_version, TT_FORK, NULL, NULL }, { "aes_AES", test_crypto_aes, TT_FORK, &passthrough_setup, (void*)"aes" }, { "aes_EVP", test_crypto_aes, TT_FORK, &passthrough_setup, (void*)"evp" }, + { "aes_ctr_testvec", test_crypto_aes_ctr_testvec, 0, NULL, NULL }, CRYPTO_LEGACY(sha), CRYPTO_LEGACY(pk), { "pk_fingerprints", test_crypto_pk_fingerprints, TT_FORK, NULL, NULL }, { "pk_base64", test_crypto_pk_base64, TT_FORK, NULL, NULL }, CRYPTO_LEGACY(digests), + { "digest_names", test_crypto_digest_names, 0, NULL, NULL }, { "sha3", test_crypto_sha3, TT_FORK, NULL, NULL}, { "sha3_xof", test_crypto_sha3_xof, TT_FORK, NULL, NULL}, CRYPTO_LEGACY(dh), @@ -2415,8 +2945,10 @@ struct testcase_t crypto_tests[] = { CRYPTO_LEGACY(base32_decode), { "kdf_TAP", test_crypto_kdf_TAP, 0, NULL, NULL }, { "hkdf_sha256", test_crypto_hkdf_sha256, 0, NULL, NULL }, + { "hkdf_sha256_testvecs", test_crypto_hkdf_sha256_testvecs, 0, NULL, NULL }, { "curve25519_impl", test_crypto_curve25519_impl, 0, NULL, NULL }, { "curve25519_impl_hibit", test_crypto_curve25519_impl, 0, NULL, (void*)"y"}, + { "curve25516_testvec", test_crypto_curve25519_testvec, 0, NULL, NULL }, { "curve25519_basepoint", test_crypto_curve25519_basepoint, TT_FORK, NULL, NULL }, { "curve25519_wrappers", test_crypto_curve25519_wrappers, 0, NULL, NULL }, @@ -2429,6 +2961,7 @@ struct testcase_t crypto_tests[] = { ED25519_TEST(blinding, 0), ED25519_TEST(testvectors, 0), ED25519_TEST(fuzz_donna, TT_FORK), + { "ed25519_storage", test_crypto_ed25519_storage, 0, NULL, NULL }, { "siphash", test_crypto_siphash, 0, NULL, NULL }, { "failure_modes", test_crypto_failure_modes, TT_FORK, NULL, NULL }, END_OF_TESTCASES diff --git a/src/test/test_data.c b/src/test/test_data.c index 32de54bc84..788489a097 100644 --- a/src/test/test_data.c +++ b/src/test/test_data.c @@ -3,6 +3,8 @@ * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +#include "test.h" + /* Our unit test expect that the AUTHORITY_CERT_* public keys will sort * in this order. */ #define AUTHORITY_CERT_A AUTHORITY_CERT_3 diff --git a/src/test/test_dir.c b/src/test/test_dir.c index 26b0e72a9a..8889ccc41b 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -11,6 +11,7 @@ #define DIRVOTE_PRIVATE #define ROUTER_PRIVATE #define ROUTERLIST_PRIVATE +#define ROUTERPARSE_PRIVATE #define HIBERNATE_PRIVATE #define NETWORKSTATUS_PRIVATE #define RELAY_PRIVATE @@ -30,6 +31,7 @@ #include "routerlist.h" #include "routerparse.h" #include "routerset.h" +#include "shared_random_state.h" #include "test.h" #include "test_dir_common.h" #include "torcert.h" @@ -192,7 +194,7 @@ test_dir_formats(void *arg) tt_assert(!crypto_pk_write_public_key_to_string(pk2 , &pk2_str, &pk2_str_len)); - /* XXXX025 router_dump_to_string should really take this from ri.*/ + /* XXXX+++ router_dump_to_string should really take this from ri.*/ options->ContactInfo = tor_strdup("Magri White " "<magri@elsewhere.example.com>"); /* Skip reachability checks for DirPort and tunnelled-dir-server */ @@ -580,7 +582,7 @@ test_dir_extrainfo_parsing(void *arg) crypto_pk_t *pk = ri->identity_pkey = crypto_pk_new(); \ tt_assert(! crypto_pk_read_public_key_from_string(pk, \ name##_KEY, strlen(name##_KEY))); \ - tt_int_op(0,OP_EQ,base16_decode(d, 20, name##_FP, strlen(name##_FP))); \ + tt_int_op(20,OP_EQ,base16_decode(d, 20, name##_FP, strlen(name##_FP))); \ digestmap_set((digestmap_t*)map, d, ri); \ ri = NULL; \ } while (0) @@ -1435,6 +1437,20 @@ test_dir_measured_bw_kb_cache(void *arg) return; } +static char * +my_dirvote_compute_params(smartlist_t *votes, int method, + int total_authorities) +{ + smartlist_t *s = dirvote_compute_params(votes, method, total_authorities); + tor_assert(s); + char *res = smartlist_join_strings(s, " ", 0, NULL); + SMARTLIST_FOREACH(s, char *, cp, tor_free(cp)); + smartlist_free(s); + return res; +} + +#define dirvote_compute_params my_dirvote_compute_params + static void test_dir_param_voting(void *arg) { @@ -1544,6 +1560,44 @@ test_dir_param_voting(void *arg) return; } +static void +test_dir_param_voting_lookup(void *arg) +{ + (void)arg; + smartlist_t *lst = smartlist_new(); + + smartlist_split_string(lst, + "moomin=9 moomin=10 moomintroll=5 fred " + "jack= electricity=sdk opa=6z abc=9 abcd=99", + NULL, 0, 0); + + tt_int_op(1000, + OP_EQ, dirvote_get_intermediate_param_value(lst, "ab", 1000)); + tt_int_op(9, OP_EQ, dirvote_get_intermediate_param_value(lst, "abc", 1000)); + tt_int_op(99, OP_EQ, + dirvote_get_intermediate_param_value(lst, "abcd", 1000)); + + /* moomin appears twice. */ + tt_int_op(-100, OP_EQ, + dirvote_get_intermediate_param_value(lst, "moomin", -100)); + /* fred and jack are truncated */ + tt_int_op(-100, OP_EQ, + dirvote_get_intermediate_param_value(lst, "fred", -100)); + tt_int_op(-100, OP_EQ, + dirvote_get_intermediate_param_value(lst, "jack", -100)); + /* electricity and opa aren't integers. */ + tt_int_op(-100, OP_EQ, + dirvote_get_intermediate_param_value(lst, "electricity", -100)); + tt_int_op(-100, OP_EQ, + dirvote_get_intermediate_param_value(lst, "opa", -100)); + + done: + SMARTLIST_FOREACH(lst, char *, cp, tor_free(cp)); + smartlist_free(lst); +} + +#undef dirvote_compute_params + /** Helper: Test that two networkstatus_voter_info_t do in fact represent the * same voting authority, and that they do in fact have all the same * information. */ @@ -1788,6 +1842,15 @@ test_routerstatus_for_v3ns(routerstatus_t *rs, time_t now) return; } +static authority_cert_t *mock_cert; + +static authority_cert_t * +get_my_v3_authority_cert_m(void) +{ + tor_assert(mock_cert); + return mock_cert; +} + /** Run a unit tests for generating and parsing networkstatuses, with * the supply test fns. */ static void @@ -1831,10 +1894,30 @@ test_a_networkstatus( tt_assert(rs_test); tt_assert(vrs_test); - tt_assert(!dir_common_authority_pk_init(&cert1, &cert2, &cert3, - &sign_skey_1, &sign_skey_2, - &sign_skey_3)); + MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); + + /* Parse certificates and keys. */ + cert1 = mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + tt_assert(cert1); + cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL); + tt_assert(cert2); + cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL); + tt_assert(cert3); + sign_skey_1 = crypto_pk_new(); + sign_skey_2 = crypto_pk_new(); + sign_skey_3 = crypto_pk_new(); sign_skey_leg1 = pk_generate(4); + sr_state_init(0, 0); + + tt_assert(!crypto_pk_read_private_key_from_string(sign_skey_1, + AUTHORITY_SIGNKEY_1, -1)); + tt_assert(!crypto_pk_read_private_key_from_string(sign_skey_2, + AUTHORITY_SIGNKEY_2, -1)); + tt_assert(!crypto_pk_read_private_key_from_string(sign_skey_3, + AUTHORITY_SIGNKEY_3, -1)); + + tt_assert(!crypto_pk_cmp_keys(sign_skey_1, cert1->signing_key)); + tt_assert(!crypto_pk_cmp_keys(sign_skey_2, cert2->signing_key)); tt_assert(!dir_common_construct_vote_1(&vote, cert1, sign_skey_1, vrs_gen, &v1, &n_vrs, now, 1)); @@ -2196,56 +2279,57 @@ test_dir_scale_bw(void *testdata) 1.0/7, 12.0, 24.0 }; - u64_dbl_t vals[8]; + double vals_dbl[8]; + uint64_t vals_u64[8]; uint64_t total; int i; (void) testdata; for (i=0; i<8; ++i) - vals[i].dbl = v[i]; + vals_dbl[i] = v[i]; - scale_array_elements_to_u64(vals, 8, &total); + scale_array_elements_to_u64(vals_u64, vals_dbl, 8, &total); tt_int_op((int)total, OP_EQ, 48); total = 0; for (i=0; i<8; ++i) { - total += vals[i].u64; + total += vals_u64[i]; } tt_assert(total >= (U64_LITERAL(1)<<60)); tt_assert(total <= (U64_LITERAL(1)<<62)); for (i=0; i<8; ++i) { /* vals[2].u64 is the scaled value of 1.0 */ - double ratio = ((double)vals[i].u64) / vals[2].u64; + double ratio = ((double)vals_u64[i]) / vals_u64[2]; tt_double_op(fabs(ratio - v[i]), OP_LT, .00001); } /* test handling of no entries */ total = 1; - scale_array_elements_to_u64(vals, 0, &total); + scale_array_elements_to_u64(vals_u64, vals_dbl, 0, &total); tt_assert(total == 0); /* make sure we don't read the array when we have no entries * may require compiler flags to catch NULL dereferences */ total = 1; - scale_array_elements_to_u64(NULL, 0, &total); + scale_array_elements_to_u64(NULL, NULL, 0, &total); tt_assert(total == 0); - scale_array_elements_to_u64(NULL, 0, NULL); + scale_array_elements_to_u64(NULL, NULL, 0, NULL); /* test handling of zero totals */ total = 1; - vals[0].dbl = 0.0; - scale_array_elements_to_u64(vals, 1, &total); + vals_dbl[0] = 0.0; + scale_array_elements_to_u64(vals_u64, vals_dbl, 1, &total); tt_assert(total == 0); - tt_assert(vals[0].u64 == 0); + tt_assert(vals_u64[0] == 0); - vals[0].dbl = 0.0; - vals[1].dbl = 0.0; - scale_array_elements_to_u64(vals, 2, NULL); - tt_assert(vals[0].u64 == 0); - tt_assert(vals[1].u64 == 0); + vals_dbl[0] = 0.0; + vals_dbl[1] = 0.0; + scale_array_elements_to_u64(vals_u64, vals_dbl, 2, NULL); + tt_assert(vals_u64[0] == 0); + tt_assert(vals_u64[1] == 0); done: ; @@ -2256,7 +2340,7 @@ test_dir_random_weighted(void *testdata) { int histogram[10]; uint64_t vals[10] = {3,1,2,4,6,0,7,5,8,9}, total=0; - u64_dbl_t inp[10]; + uint64_t inp_u64[10]; int i, choice; const int n = 50000; double max_sq_error; @@ -2266,12 +2350,12 @@ test_dir_random_weighted(void *testdata) * in a scrambled order to make sure we don't depend on order. */ memset(histogram,0,sizeof(histogram)); for (i=0; i<10; ++i) { - inp[i].u64 = vals[i]; + inp_u64[i] = vals[i]; total += vals[i]; } tt_u64_op(total, OP_EQ, 45); for (i=0; i<n; ++i) { - choice = choose_array_element_by_weight(inp, 10); + choice = choose_array_element_by_weight(inp_u64, 10); tt_int_op(choice, OP_GE, 0); tt_int_op(choice, OP_LT, 10); histogram[choice]++; @@ -2298,16 +2382,16 @@ test_dir_random_weighted(void *testdata) /* Now try a singleton; do we choose it? */ for (i = 0; i < 100; ++i) { - choice = choose_array_element_by_weight(inp, 1); + choice = choose_array_element_by_weight(inp_u64, 1); tt_int_op(choice, OP_EQ, 0); } /* Now try an array of zeros. We should choose randomly. */ memset(histogram,0,sizeof(histogram)); for (i = 0; i < 5; ++i) - inp[i].u64 = 0; + inp_u64[i] = 0; for (i = 0; i < n; ++i) { - choice = choose_array_element_by_weight(inp, 5); + choice = choose_array_element_by_weight(inp_u64, 5); tt_int_op(choice, OP_GE, 0); tt_int_op(choice, OP_LT, 5); histogram[choice]++; @@ -2847,7 +2931,7 @@ test_dir_dirserv_set_routerstatus_testing(void *arg) (void)arg; /* Init options */ - mock_options = malloc(sizeof(or_options_t)); + mock_options = tor_malloc(sizeof(or_options_t)); reset_options(mock_options, &mock_get_options_calls); MOCK(get_options, mock_get_options); @@ -2865,10 +2949,10 @@ test_dir_dirserv_set_routerstatus_testing(void *arg) routerset_parse(routerset_none, ROUTERSET_NONE_STR, "No routers"); /* Init routerstatuses */ - routerstatus_t *rs_a = malloc(sizeof(routerstatus_t)); + routerstatus_t *rs_a = tor_malloc(sizeof(routerstatus_t)); reset_routerstatus(rs_a, ROUTER_A_ID_STR, ROUTER_A_IPV4); - routerstatus_t *rs_b = malloc(sizeof(routerstatus_t)); + routerstatus_t *rs_b = tor_malloc(sizeof(routerstatus_t)); reset_routerstatus(rs_b, ROUTER_B_ID_STR, ROUTER_B_IPV4); /* Sanity check that routersets correspond to routerstatuses. @@ -3053,7 +3137,7 @@ test_dir_dirserv_set_routerstatus_testing(void *arg) tt_assert(rs_b->is_hs_dir == 1); done: - free(mock_options); + tor_free(mock_options); mock_options = NULL; UNMOCK(get_options); @@ -3062,8 +3146,8 @@ test_dir_dirserv_set_routerstatus_testing(void *arg) routerset_free(routerset_a); routerset_free(routerset_none); - free(rs_a); - free(rs_b); + tor_free(rs_a); + tor_free(rs_b); } static void @@ -3332,13 +3416,16 @@ test_dir_download_status_schedule(void *arg) (void)arg; download_status_t dls_failure = { 0, 0, 0, DL_SCHED_GENERIC, DL_WANT_AUTHORITY, - DL_SCHED_INCREMENT_FAILURE }; + DL_SCHED_INCREMENT_FAILURE, + DL_SCHED_DETERMINISTIC, 0, 0 }; download_status_t dls_attempt = { 0, 0, 0, DL_SCHED_CONSENSUS, DL_WANT_ANY_DIRSERVER, - DL_SCHED_INCREMENT_ATTEMPT}; + DL_SCHED_INCREMENT_ATTEMPT, + DL_SCHED_DETERMINISTIC, 0, 0 }; download_status_t dls_bridge = { 0, 0, 0, DL_SCHED_BRIDGE, DL_WANT_AUTHORITY, - DL_SCHED_INCREMENT_FAILURE}; + DL_SCHED_INCREMENT_FAILURE, + DL_SCHED_DETERMINISTIC, 0, 0 }; int increment = -1; int expected_increment = -1; time_t current_time = time(NULL); @@ -3354,6 +3441,7 @@ test_dir_download_status_schedule(void *arg) delay1 = 1000; increment = download_status_schedule_get_delay(&dls_failure, schedule, + 0, INT_MAX, TIME_MIN); expected_increment = delay1; tt_assert(increment == expected_increment); @@ -3362,6 +3450,7 @@ test_dir_download_status_schedule(void *arg) delay1 = INT_MAX; increment = download_status_schedule_get_delay(&dls_failure, schedule, + 0, INT_MAX, -1); expected_increment = delay1; tt_assert(increment == expected_increment); @@ -3370,6 +3459,7 @@ test_dir_download_status_schedule(void *arg) delay1 = 0; increment = download_status_schedule_get_delay(&dls_attempt, schedule, + 0, INT_MAX, 0); expected_increment = delay1; tt_assert(increment == expected_increment); @@ -3378,6 +3468,7 @@ test_dir_download_status_schedule(void *arg) delay1 = 1000; increment = download_status_schedule_get_delay(&dls_attempt, schedule, + 0, INT_MAX, 1); expected_increment = delay1; tt_assert(increment == expected_increment); @@ -3386,6 +3477,7 @@ test_dir_download_status_schedule(void *arg) delay1 = INT_MAX; increment = download_status_schedule_get_delay(&dls_bridge, schedule, + 0, INT_MAX, current_time); expected_increment = delay1; tt_assert(increment == expected_increment); @@ -3394,6 +3486,7 @@ test_dir_download_status_schedule(void *arg) delay1 = 1; increment = download_status_schedule_get_delay(&dls_bridge, schedule, + 0, INT_MAX, TIME_MAX); expected_increment = delay1; tt_assert(increment == expected_increment); @@ -3406,6 +3499,7 @@ test_dir_download_status_schedule(void *arg) delay2 = 100; increment = download_status_schedule_get_delay(&dls_attempt, schedule, + 0, INT_MAX, current_time); expected_increment = delay2; tt_assert(increment == expected_increment); @@ -3414,6 +3508,7 @@ test_dir_download_status_schedule(void *arg) delay2 = 1; increment = download_status_schedule_get_delay(&dls_bridge, schedule, + 0, INT_MAX, current_time); expected_increment = delay2; tt_assert(increment == expected_increment); @@ -3426,6 +3521,7 @@ test_dir_download_status_schedule(void *arg) delay2 = 5; increment = download_status_schedule_get_delay(&dls_attempt, schedule, + 0, INT_MAX, current_time); expected_increment = delay2; tt_assert(increment == expected_increment); @@ -3434,6 +3530,7 @@ test_dir_download_status_schedule(void *arg) delay2 = 17; increment = download_status_schedule_get_delay(&dls_bridge, schedule, + 0, INT_MAX, current_time); expected_increment = delay2; tt_assert(increment == expected_increment); @@ -3446,6 +3543,7 @@ test_dir_download_status_schedule(void *arg) delay2 = 35; increment = download_status_schedule_get_delay(&dls_attempt, schedule, + 0, INT_MAX, current_time); expected_increment = INT_MAX; tt_assert(increment == expected_increment); @@ -3454,6 +3552,7 @@ test_dir_download_status_schedule(void *arg) delay2 = 99; increment = download_status_schedule_get_delay(&dls_bridge, schedule, + 0, INT_MAX, current_time); expected_increment = INT_MAX; tt_assert(increment == expected_increment); @@ -3465,15 +3564,58 @@ test_dir_download_status_schedule(void *arg) } static void +test_dir_download_status_random_backoff(void *arg) +{ + download_status_t dls_random = + { 0, 0, 0, DL_SCHED_GENERIC, DL_WANT_AUTHORITY, + DL_SCHED_INCREMENT_FAILURE, DL_SCHED_RANDOM_EXPONENTIAL, 0, 0 }; + int increment = -1; + int old_increment; + time_t current_time = time(NULL); + const int min_delay = 0; + const int max_delay = 1000000; + + (void)arg; + + /* Check the random backoff cases */ + old_increment = 0; + do { + increment = download_status_schedule_get_delay(&dls_random, + NULL, + min_delay, max_delay, + current_time); + /* Test */ + tt_int_op(increment, OP_GE, min_delay); + tt_int_op(increment, OP_LE, max_delay); + tt_int_op(increment, OP_GE, old_increment); + /* We at most double, and maybe add one */ + tt_int_op(increment, OP_LE, 2 * old_increment + 1); + + /* Advance */ + current_time += increment; + ++(dls_random.n_download_attempts); + ++(dls_random.n_download_failures); + + /* Try another maybe */ + old_increment = increment; + } while (increment < max_delay); + + done: + return; +} + +static void test_dir_download_status_increment(void *arg) { (void)arg; download_status_t dls_failure = { 0, 0, 0, DL_SCHED_GENERIC, DL_WANT_AUTHORITY, - DL_SCHED_INCREMENT_FAILURE }; + DL_SCHED_INCREMENT_FAILURE, + DL_SCHED_DETERMINISTIC, 0, 0 }; download_status_t dls_attempt = { 0, 0, 0, DL_SCHED_BRIDGE, DL_WANT_ANY_DIRSERVER, - DL_SCHED_INCREMENT_ATTEMPT}; + DL_SCHED_INCREMENT_ATTEMPT, + DL_SCHED_DETERMINISTIC, 0, 0 }; int delay0 = -1; int delay1 = -1; int delay2 = -1; @@ -4042,6 +4184,1003 @@ test_dir_choose_compression_level(void* data) done: ; } +/* + * Mock check_private_dir(), and always succeed - no need to actually + * look at or create anything on the filesystem. + */ + +static int +mock_check_private_dir(const char *dirname, cpd_check_t check, + const char *effective_user) +{ + (void)dirname; + (void)check; + (void)effective_user; + + return 0; +} + +/* + * This really mocks options_get_datadir_fname2_suffix(), but for testing + * dump_desc(), we only care about get_datadir_fname(sub1), which is defined + * in config.h as: + * + * options_get_datadir_fname2_suffix(get_options(), sub1, NULL, NULL) + */ + +static char * +mock_get_datadir_fname(const or_options_t *options, + const char *sub1, const char *sub2, + const char *suffix) +{ + char *rv = NULL; + + /* + * Assert we were called like get_datadir_fname2() or get_datadir_fname(), + * since that's all we implement here. + */ + tt_assert(options != NULL); + tt_assert(sub1 != NULL); + /* + * No particular assertions about sub2, since we could be in the + * get_datadir_fname() or get_datadir_fname2() case. + */ + tt_assert(suffix == NULL); + + /* Just duplicate the basename and return it for this mock */ + if (sub2) { + /* If we have sub2, it's the basename, otherwise sub1 */ + rv = tor_strdup(sub2); + } else { + rv = tor_strdup(sub1); + } + + done: + return rv; +} + +static char *last_unlinked_path = NULL; +static int unlinked_count = 0; + +static void +mock_unlink_reset(void) +{ + tor_free(last_unlinked_path); + unlinked_count = 0; +} + +static int +mock_unlink(const char *path) +{ + tt_assert(path != NULL); + + tor_free(last_unlinked_path); + last_unlinked_path = tor_strdup(path); + ++unlinked_count; + + done: + return 0; +} + +static char *last_write_str_path = NULL; +static uint8_t last_write_str_hash[DIGEST256_LEN]; +static int write_str_count = 0; + +static void +mock_write_str_to_file_reset(void) +{ + tor_free(last_write_str_path); + write_str_count = 0; +} + +static int +mock_write_str_to_file(const char *path, const char *str, int bin) +{ + size_t len; + uint8_t hash[DIGEST256_LEN]; + + (void)bin; + + tt_assert(path != NULL); + tt_assert(str != NULL); + + len = strlen(str); + crypto_digest256((char *)hash, str, len, DIGEST_SHA256); + + tor_free(last_write_str_path); + last_write_str_path = tor_strdup(path); + memcpy(last_write_str_hash, hash, sizeof(last_write_str_hash)); + ++write_str_count; + + done: + return 0; +} + +static void +test_dir_dump_unparseable_descriptors(void *data) +{ + /* + * These bogus descriptors look nothing at all like real bogus descriptors + * we might see, but we're only testing dump_desc() here, not the parser. + */ + const char *test_desc_type = "squamous"; + /* strlen(test_desc_1) = 583 bytes */ + const char *test_desc_1 = + "The most merciful thing in the world, I think, is the inability of the " + "human mind to correlate all its contents. We live on a placid island of" + " ignorance in the midst of black seas of infinity, and it was not meant" + " that we should voyage far. The sciences, each straining in its own dir" + "ection, have hitherto harmed us little; but some day the piecing togeth" + "er of dissociated knowledge will open up such terrifying vistas of real" + "ity, and of our frightful position therein, that we shall either go mad" + "from the revelation or flee from the light into the peace and safety of" + "a new dark age."; + uint8_t test_desc_1_hash[DIGEST256_LEN]; + char test_desc_1_hash_str[HEX_DIGEST256_LEN+1]; + /* strlen(test_desc_2) = 650 bytes */ + const char *test_desc_2 = + "I think their predominant colour was a greyish-green, though they had w" + "hite bellies. They were mostly shiny and slippery, but the ridges of th" + "eir backs were scaly. Their forms vaguely suggested the anthropoid, whi" + "le their heads were the heads of fish, with prodigious bulging eyes tha" + "t never closed. At the sides of their necks were palpitating gills, and" + "their long paws were webbed. They hopped irregularly, sometimes on two " + "legs and sometimes on four. I was somehow glad that they had no more th" + "an four limbs. Their croaking, baying voices, clearly wed tar articulat" + "e speech, held all the dark shades of expression which their staring fa" + "ces lacked."; + uint8_t test_desc_2_hash[DIGEST256_LEN]; + char test_desc_2_hash_str[HEX_DIGEST256_LEN+1]; + /* strlen(test_desc_3) = 700 bytes */ + const char *test_desc_3 = + "Without knowing what futurism is like, Johansen achieved something very" + "close to it when he spoke of the city; for instead of describing any de" + "finite structure or building, he dwells only on broad impressions of va" + "st angles and stone surfaces - surfaces too great to belong to anything" + "right or proper for this earth, and impious with horrible images and hi" + "eroglyphs. I mention his talk about angles because it suggests somethin" + "g Wilcox had told me of his awful dreams. He said that the geometry of " + "the dream-place he saw was abnormal, non-Euclidean, and loathsomely red" + "olent of spheres and dimensions apart from ours. Now an unlettered seam" + "an felt the same thing whilst gazing at the terrible reality."; + uint8_t test_desc_3_hash[DIGEST256_LEN]; + char test_desc_3_hash_str[HEX_DIGEST256_LEN+1]; + /* strlen(test_desc_3) = 604 bytes */ + const char *test_desc_4 = + "So we glanced back simultaneously, it would appear; though no doubt the" + "incipient motion of one prompted the imitation of the other. As we did " + "so we flashed both torches full strength at the momentarily thinned mis" + "t; either from sheer primitive anxiety to see all we could, or in a les" + "s primitive but equally unconscious effort to dazzle the entity before " + "we dimmed our light and dodged among the penguins of the labyrinth cent" + "er ahead. Unhappy act! Not Orpheus himself, or Lot's wife, paid much mo" + "re dearly for a backward glance. And again came that shocking, wide-ran" + "ged piping - \"Tekeli-li! Tekeli-li!\""; + uint8_t test_desc_4_hash[DIGEST256_LEN]; + char test_desc_4_hash_str[HEX_DIGEST256_LEN+1]; + (void)data; + + /* + * Set up options mock so we can force a tiny FIFO size and generate + * cleanups. + */ + mock_options = tor_malloc(sizeof(or_options_t)); + reset_options(mock_options, &mock_get_options_calls); + mock_options->MaxUnparseableDescSizeToLog = 1536; + MOCK(get_options, mock_get_options); + MOCK(check_private_dir, mock_check_private_dir); + MOCK(options_get_datadir_fname2_suffix, + mock_get_datadir_fname); + + /* + * Set up unlink and write mocks + */ + MOCK(tor_unlink, mock_unlink); + mock_unlink_reset(); + MOCK(write_str_to_file, mock_write_str_to_file); + mock_write_str_to_file_reset(); + + /* + * Compute hashes we'll need to recognize which descriptor is which + */ + crypto_digest256((char *)test_desc_1_hash, test_desc_1, + strlen(test_desc_1), DIGEST_SHA256); + base16_encode(test_desc_1_hash_str, sizeof(test_desc_1_hash_str), + (const char *)test_desc_1_hash, + sizeof(test_desc_1_hash)); + crypto_digest256((char *)test_desc_2_hash, test_desc_2, + strlen(test_desc_2), DIGEST_SHA256); + base16_encode(test_desc_2_hash_str, sizeof(test_desc_2_hash_str), + (const char *)test_desc_2_hash, + sizeof(test_desc_2_hash)); + crypto_digest256((char *)test_desc_3_hash, test_desc_3, + strlen(test_desc_3), DIGEST_SHA256); + base16_encode(test_desc_3_hash_str, sizeof(test_desc_3_hash_str), + (const char *)test_desc_3_hash, + sizeof(test_desc_3_hash)); + crypto_digest256((char *)test_desc_4_hash, test_desc_4, + strlen(test_desc_4), DIGEST_SHA256); + base16_encode(test_desc_4_hash_str, sizeof(test_desc_4_hash_str), + (const char *)test_desc_4_hash, + sizeof(test_desc_4_hash)); + + /* + * Reset the FIFO and check its state + */ + dump_desc_fifo_cleanup(); + tt_u64_op(len_descs_dumped, ==, 0); + tt_assert(descs_dumped == NULL || smartlist_len(descs_dumped) == 0); + + /* + * (1) Fire off dump_desc() once; these descriptors should all be safely + * smaller than configured FIFO size. + */ + + dump_desc(test_desc_1, test_desc_type); + + /* + * Assert things about the FIFO state + */ + tt_u64_op(len_descs_dumped, ==, strlen(test_desc_1)); + tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 1); + + /* + * Assert things about the mocks + */ + tt_int_op(unlinked_count, ==, 0); + tt_int_op(write_str_count, ==, 1); + tt_mem_op(last_write_str_hash, OP_EQ, test_desc_1_hash, DIGEST_SHA256); + + /* + * Reset the FIFO and check its state + */ + dump_desc_fifo_cleanup(); + tt_u64_op(len_descs_dumped, ==, 0); + tt_assert(descs_dumped == NULL || smartlist_len(descs_dumped) == 0); + + /* + * Reset the mocks and check their state + */ + mock_unlink_reset(); + mock_write_str_to_file_reset(); + tt_int_op(unlinked_count, ==, 0); + tt_int_op(write_str_count, ==, 0); + + /* + * (2) Fire off dump_desc() twice; this still should trigger no cleanup. + */ + + /* First time */ + dump_desc(test_desc_2, test_desc_type); + + /* + * Assert things about the FIFO state + */ + tt_u64_op(len_descs_dumped, ==, strlen(test_desc_2)); + tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 1); + + /* + * Assert things about the mocks + */ + tt_int_op(unlinked_count, ==, 0); + tt_int_op(write_str_count, ==, 1); + tt_mem_op(last_write_str_hash, OP_EQ, test_desc_2_hash, DIGEST_SHA256); + + /* Second time */ + dump_desc(test_desc_3, test_desc_type); + + /* + * Assert things about the FIFO state + */ + tt_u64_op(len_descs_dumped, ==, strlen(test_desc_2) + strlen(test_desc_3)); + tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2); + + /* + * Assert things about the mocks + */ + tt_int_op(unlinked_count, ==, 0); + tt_int_op(write_str_count, ==, 2); + tt_mem_op(last_write_str_hash, OP_EQ, test_desc_3_hash, DIGEST_SHA256); + + /* + * Reset the FIFO and check its state + */ + dump_desc_fifo_cleanup(); + tt_u64_op(len_descs_dumped, ==, 0); + tt_assert(descs_dumped == NULL || smartlist_len(descs_dumped) == 0); + + /* + * Reset the mocks and check their state + */ + mock_unlink_reset(); + mock_write_str_to_file_reset(); + tt_int_op(unlinked_count, ==, 0); + tt_int_op(write_str_count, ==, 0); + + /* + * (3) Three calls to dump_desc cause a FIFO cleanup + */ + + /* First time */ + dump_desc(test_desc_4, test_desc_type); + + /* + * Assert things about the FIFO state + */ + tt_u64_op(len_descs_dumped, ==, strlen(test_desc_4)); + tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 1); + + /* + * Assert things about the mocks + */ + tt_int_op(unlinked_count, ==, 0); + tt_int_op(write_str_count, ==, 1); + tt_mem_op(last_write_str_hash, OP_EQ, test_desc_4_hash, DIGEST_SHA256); + + /* Second time */ + dump_desc(test_desc_1, test_desc_type); + + /* + * Assert things about the FIFO state + */ + tt_u64_op(len_descs_dumped, ==, strlen(test_desc_4) + strlen(test_desc_1)); + tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2); + + /* + * Assert things about the mocks + */ + tt_int_op(unlinked_count, ==, 0); + tt_int_op(write_str_count, ==, 2); + tt_mem_op(last_write_str_hash, OP_EQ, test_desc_1_hash, DIGEST_SHA256); + + /* Third time - we should unlink the dump of test_desc_4 here */ + dump_desc(test_desc_2, test_desc_type); + + /* + * Assert things about the FIFO state + */ + tt_u64_op(len_descs_dumped, ==, strlen(test_desc_1) + strlen(test_desc_2)); + tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2); + + /* + * Assert things about the mocks + */ + tt_int_op(unlinked_count, ==, 1); + tt_int_op(write_str_count, ==, 3); + tt_mem_op(last_write_str_hash, OP_EQ, test_desc_2_hash, DIGEST_SHA256); + + /* + * Reset the FIFO and check its state + */ + dump_desc_fifo_cleanup(); + tt_u64_op(len_descs_dumped, ==, 0); + tt_assert(descs_dumped == NULL || smartlist_len(descs_dumped) == 0); + + /* + * Reset the mocks and check their state + */ + mock_unlink_reset(); + mock_write_str_to_file_reset(); + tt_int_op(unlinked_count, ==, 0); + tt_int_op(write_str_count, ==, 0); + + /* + * (4) But repeating one (A B B) doesn't overflow and cleanup + */ + + /* First time */ + dump_desc(test_desc_3, test_desc_type); + + /* + * Assert things about the FIFO state + */ + tt_u64_op(len_descs_dumped, ==, strlen(test_desc_3)); + tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 1); + + /* + * Assert things about the mocks + */ + tt_int_op(unlinked_count, ==, 0); + tt_int_op(write_str_count, ==, 1); + tt_mem_op(last_write_str_hash, OP_EQ, test_desc_3_hash, DIGEST_SHA256); + + /* Second time */ + dump_desc(test_desc_4, test_desc_type); + + /* + * Assert things about the FIFO state + */ + tt_u64_op(len_descs_dumped, ==, strlen(test_desc_3) + strlen(test_desc_4)); + tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2); + + /* + * Assert things about the mocks + */ + tt_int_op(unlinked_count, ==, 0); + tt_int_op(write_str_count, ==, 2); + tt_mem_op(last_write_str_hash, OP_EQ, test_desc_4_hash, DIGEST_SHA256); + + /* Third time */ + dump_desc(test_desc_4, test_desc_type); + + /* + * Assert things about the FIFO state + */ + tt_u64_op(len_descs_dumped, ==, strlen(test_desc_3) + strlen(test_desc_4)); + tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2); + + /* + * Assert things about the mocks + */ + tt_int_op(unlinked_count, ==, 0); + tt_int_op(write_str_count, ==, 2); + tt_mem_op(last_write_str_hash, OP_EQ, test_desc_4_hash, DIGEST_SHA256); + + /* + * Reset the FIFO and check its state + */ + dump_desc_fifo_cleanup(); + tt_u64_op(len_descs_dumped, ==, 0); + tt_assert(descs_dumped == NULL || smartlist_len(descs_dumped) == 0); + + /* + * Reset the mocks and check their state + */ + mock_unlink_reset(); + mock_write_str_to_file_reset(); + tt_int_op(unlinked_count, ==, 0); + tt_int_op(write_str_count, ==, 0); + + /* + * (5) Same for the (A B A) repetition + */ + + /* First time */ + dump_desc(test_desc_1, test_desc_type); + + /* + * Assert things about the FIFO state + */ + tt_u64_op(len_descs_dumped, ==, strlen(test_desc_1)); + tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 1); + + /* + * Assert things about the mocks + */ + tt_int_op(unlinked_count, ==, 0); + tt_int_op(write_str_count, ==, 1); + tt_mem_op(last_write_str_hash, OP_EQ, test_desc_1_hash, DIGEST_SHA256); + + /* Second time */ + dump_desc(test_desc_2, test_desc_type); + + /* + * Assert things about the FIFO state + */ + tt_u64_op(len_descs_dumped, ==, strlen(test_desc_1) + strlen(test_desc_2)); + tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2); + + /* + * Assert things about the mocks + */ + tt_int_op(unlinked_count, ==, 0); + tt_int_op(write_str_count, ==, 2); + tt_mem_op(last_write_str_hash, OP_EQ, test_desc_2_hash, DIGEST_SHA256); + + /* Third time */ + dump_desc(test_desc_1, test_desc_type); + + /* + * Assert things about the FIFO state + */ + tt_u64_op(len_descs_dumped, ==, strlen(test_desc_1) + strlen(test_desc_2)); + tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2); + + /* + * Assert things about the mocks + */ + tt_int_op(unlinked_count, ==, 0); + tt_int_op(write_str_count, ==, 2); + tt_mem_op(last_write_str_hash, OP_EQ, test_desc_2_hash, DIGEST_SHA256); + + /* + * Reset the FIFO and check its state + */ + dump_desc_fifo_cleanup(); + tt_u64_op(len_descs_dumped, ==, 0); + tt_assert(descs_dumped == NULL || smartlist_len(descs_dumped) == 0); + + /* + * Reset the mocks and check their state + */ + mock_unlink_reset(); + mock_write_str_to_file_reset(); + tt_int_op(unlinked_count, ==, 0); + tt_int_op(write_str_count, ==, 0); + + /* + * (6) (A B B C) triggering overflow on C causes A, not B to be unlinked + */ + + /* First time */ + dump_desc(test_desc_3, test_desc_type); + + /* + * Assert things about the FIFO state + */ + tt_u64_op(len_descs_dumped, ==, strlen(test_desc_3)); + tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 1); + + /* + * Assert things about the mocks + */ + tt_int_op(unlinked_count, ==, 0); + tt_int_op(write_str_count, ==, 1); + tt_mem_op(last_write_str_hash, OP_EQ, test_desc_3_hash, DIGEST_SHA256); + + /* Second time */ + dump_desc(test_desc_4, test_desc_type); + + /* + * Assert things about the FIFO state + */ + tt_u64_op(len_descs_dumped, ==, strlen(test_desc_3) + strlen(test_desc_4)); + tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2); + + /* + * Assert things about the mocks + */ + tt_int_op(unlinked_count, ==, 0); + tt_int_op(write_str_count, ==, 2); + tt_mem_op(last_write_str_hash, OP_EQ, test_desc_4_hash, DIGEST_SHA256); + + /* Third time */ + dump_desc(test_desc_4, test_desc_type); + + /* + * Assert things about the FIFO state + */ + tt_u64_op(len_descs_dumped, ==, strlen(test_desc_3) + strlen(test_desc_4)); + tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2); + + /* + * Assert things about the mocks + */ + tt_int_op(unlinked_count, ==, 0); + tt_int_op(write_str_count, ==, 2); + tt_mem_op(last_write_str_hash, OP_EQ, test_desc_4_hash, DIGEST_SHA256); + + /* Fourth time - we should unlink the dump of test_desc_3 here */ + dump_desc(test_desc_1, test_desc_type); + + /* + * Assert things about the FIFO state + */ + tt_u64_op(len_descs_dumped, ==, strlen(test_desc_4) + strlen(test_desc_1)); + tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2); + + /* + * Assert things about the mocks + */ + tt_int_op(unlinked_count, ==, 1); + tt_int_op(write_str_count, ==, 3); + tt_mem_op(last_write_str_hash, OP_EQ, test_desc_1_hash, DIGEST_SHA256); + + /* + * Reset the FIFO and check its state + */ + dump_desc_fifo_cleanup(); + tt_u64_op(len_descs_dumped, ==, 0); + tt_assert(descs_dumped == NULL || smartlist_len(descs_dumped) == 0); + + /* + * Reset the mocks and check their state + */ + mock_unlink_reset(); + mock_write_str_to_file_reset(); + tt_int_op(unlinked_count, ==, 0); + tt_int_op(write_str_count, ==, 0); + + /* + * (7) (A B A C) triggering overflow on C causes B, not A to be unlinked + */ + + /* First time */ + dump_desc(test_desc_2, test_desc_type); + + /* + * Assert things about the FIFO state + */ + tt_u64_op(len_descs_dumped, ==, strlen(test_desc_2)); + tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 1); + + /* + * Assert things about the mocks + */ + tt_int_op(unlinked_count, ==, 0); + tt_int_op(write_str_count, ==, 1); + tt_mem_op(last_write_str_hash, OP_EQ, test_desc_2_hash, DIGEST_SHA256); + + /* Second time */ + dump_desc(test_desc_3, test_desc_type); + + /* + * Assert things about the FIFO state + */ + tt_u64_op(len_descs_dumped, ==, strlen(test_desc_2) + strlen(test_desc_3)); + tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2); + + /* + * Assert things about the mocks + */ + tt_int_op(unlinked_count, ==, 0); + tt_int_op(write_str_count, ==, 2); + tt_mem_op(last_write_str_hash, OP_EQ, test_desc_3_hash, DIGEST_SHA256); + + /* Third time */ + dump_desc(test_desc_2, test_desc_type); + + /* + * Assert things about the FIFO state + */ + tt_u64_op(len_descs_dumped, ==, strlen(test_desc_2) + strlen(test_desc_3)); + tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2); + + /* + * Assert things about the mocks + */ + tt_int_op(unlinked_count, ==, 0); + tt_int_op(write_str_count, ==, 2); + tt_mem_op(last_write_str_hash, OP_EQ, test_desc_3_hash, DIGEST_SHA256); + + /* Fourth time - we should unlink the dump of test_desc_3 here */ + dump_desc(test_desc_4, test_desc_type); + + /* + * Assert things about the FIFO state + */ + tt_u64_op(len_descs_dumped, ==, strlen(test_desc_2) + strlen(test_desc_4)); + tt_assert(descs_dumped != NULL && smartlist_len(descs_dumped) == 2); + + /* + * Assert things about the mocks + */ + tt_int_op(unlinked_count, ==, 1); + tt_int_op(write_str_count, ==, 3); + tt_mem_op(last_write_str_hash, OP_EQ, test_desc_4_hash, DIGEST_SHA256); + + /* + * Reset the FIFO and check its state + */ + dump_desc_fifo_cleanup(); + tt_u64_op(len_descs_dumped, ==, 0); + tt_assert(descs_dumped == NULL || smartlist_len(descs_dumped) == 0); + + /* + * Reset the mocks and check their state + */ + mock_unlink_reset(); + mock_write_str_to_file_reset(); + tt_int_op(unlinked_count, ==, 0); + tt_int_op(write_str_count, ==, 0); + + done: + + /* Clean up the fifo */ + dump_desc_fifo_cleanup(); + + /* Remove mocks */ + UNMOCK(tor_unlink); + mock_unlink_reset(); + UNMOCK(write_str_to_file); + mock_write_str_to_file_reset(); + UNMOCK(options_get_datadir_fname2_suffix); + UNMOCK(check_private_dir); + UNMOCK(get_options); + tor_free(mock_options); + mock_options = NULL; + + return; +} + +/* Variables for reset_read_file_to_str_mock() */ + +static int enforce_expected_filename = 0; +static char *expected_filename = NULL; +static char *file_content = NULL; +static size_t file_content_len = 0; +static struct stat file_stat; +static int read_count = 0, read_call_count = 0; + +static void +reset_read_file_to_str_mock(void) +{ + tor_free(expected_filename); + tor_free(file_content); + file_content_len = 0; + memset(&file_stat, 0, sizeof(file_stat)); + read_count = 0; + read_call_count = 0; +} + +static char * +read_file_to_str_mock(const char *filename, int flags, + struct stat *stat_out) { + char *result = NULL; + + /* Insist we got a filename */ + tt_assert(filename != NULL); + + /* We ignore flags */ + (void)flags; + + /* Bump the call count */ + ++read_call_count; + + if (enforce_expected_filename) { + tt_assert(expected_filename); + tt_str_op(filename, OP_EQ, expected_filename); + } + + if (expected_filename != NULL && + file_content != NULL && + strcmp(filename, expected_filename) == 0) { + /* You asked for it, you got it */ + + /* + * This is the same behavior as the real read_file_to_str(); + * if there's a NUL, the real size ends up in stat_out. + */ + result = tor_malloc(file_content_len + 1); + if (file_content_len > 0) { + memcpy(result, file_content, file_content_len); + } + result[file_content_len] = '\0'; + + /* Do we need to set up stat_out? */ + if (stat_out != NULL) { + memcpy(stat_out, &file_stat, sizeof(file_stat)); + /* We always return the correct length here */ + stat_out->st_size = file_content_len; + } + + /* Wooo, we have a return value - bump the counter */ + ++read_count; + } + /* else no match, return NULL */ + + done: + return result; +} + +/* This one tests dump_desc_populate_one_file() */ +static void +test_dir_populate_dump_desc_fifo(void *data) +{ + const char *dirname = "foo"; + const char *fname = NULL; + dumped_desc_t *ent; + + (void)data; + + /* + * Set up unlink and read_file_to_str mocks + */ + MOCK(tor_unlink, mock_unlink); + mock_unlink_reset(); + MOCK(read_file_to_str, read_file_to_str_mock); + reset_read_file_to_str_mock(); + + /* Check state of unlink mock */ + tt_int_op(unlinked_count, ==, 0); + + /* Some cases that should fail before trying to read the file */ + ent = dump_desc_populate_one_file(dirname, "bar"); + tt_assert(ent == NULL); + tt_int_op(unlinked_count, ==, 1); + tt_int_op(read_count, ==, 0); + tt_int_op(read_call_count, ==, 0); + + ent = dump_desc_populate_one_file(dirname, "unparseable-desc"); + tt_assert(ent == NULL); + tt_int_op(unlinked_count, ==, 2); + tt_int_op(read_count, ==, 0); + tt_int_op(read_call_count, ==, 0); + + ent = dump_desc_populate_one_file(dirname, "unparseable-desc.baz"); + tt_assert(ent == NULL); + tt_int_op(unlinked_count, ==, 3); + tt_int_op(read_count, ==, 0); + tt_int_op(read_call_count, ==, 0); + + ent = dump_desc_populate_one_file( + dirname, + "unparseable-desc.08AE85E90461F59E"); + tt_assert(ent == NULL); + tt_int_op(unlinked_count, ==, 4); + tt_int_op(read_count, ==, 0); + tt_int_op(read_call_count, ==, 0); + + ent = dump_desc_populate_one_file( + dirname, + "unparseable-desc.08AE85E90461F59EDF0981323F3A70D02B55AB54B44B04F" + "287D72F7B72F242E85C8CB0EDA8854A99"); + tt_assert(ent == NULL); + tt_int_op(unlinked_count, ==, 5); + tt_int_op(read_count, ==, 0); + tt_int_op(read_call_count, ==, 0); + + /* This is a correct-length digest but base16_decode() will fail */ + ent = dump_desc_populate_one_file( + dirname, + "unparseable-desc.68219B8BGE64B705A6FFC728C069DC596216D60A7D7520C" + "D5ECE250D912E686B"); + tt_assert(ent == NULL); + tt_int_op(unlinked_count, ==, 6); + tt_int_op(read_count, ==, 0); + tt_int_op(read_call_count, ==, 0); + + /* This one has a correctly formed filename and should try reading */ + + /* Read fails */ + ent = dump_desc_populate_one_file( + dirname, + "unparseable-desc.DF0981323F3A70D02B55AB54B44B04F287D72F7B72F242E" + "85C8CB0EDA8854A99"); + tt_assert(ent == NULL); + tt_int_op(unlinked_count, ==, 7); + tt_int_op(read_count, ==, 0); + tt_int_op(read_call_count, ==, 1); + + /* This read will succeed but the digest won't match the file content */ + fname = + "unparseable-desc." + "DF0981323F3A70D02B55AB54B44B04F287D72F7B72F242E85C8CB0EDA8854A99"; + enforce_expected_filename = 1; + tor_asprintf(&expected_filename, "%s%s%s", dirname, PATH_SEPARATOR, fname); + file_content = tor_strdup("hanc culpam maiorem an illam dicam?"); + file_content_len = strlen(file_content); + file_stat.st_mtime = 123456; + ent = dump_desc_populate_one_file(dirname, fname); + enforce_expected_filename = 0; + tt_assert(ent == NULL); + tt_int_op(unlinked_count, ==, 8); + tt_int_op(read_count, ==, 1); + tt_int_op(read_call_count, ==, 2); + tor_free(expected_filename); + tor_free(file_content); + + /* This one will match */ + fname = + "unparseable-desc." + "0786C7173447B7FB033FFCA2FC47C3CF71C30DD47CA8236D3FC7FF35853271C6"; + tor_asprintf(&expected_filename, "%s%s%s", dirname, PATH_SEPARATOR, fname); + file_content = tor_strdup("hanc culpam maiorem an illam dicam?"); + file_content_len = strlen(file_content); + file_stat.st_mtime = 789012; + ent = dump_desc_populate_one_file(dirname, fname); + tt_assert(ent != NULL); + tt_int_op(unlinked_count, ==, 8); + tt_int_op(read_count, ==, 2); + tt_int_op(read_call_count, ==, 3); + tt_str_op(ent->filename, OP_EQ, expected_filename); + tt_int_op(ent->len, ==, file_content_len); + tt_int_op(ent->when, ==, file_stat.st_mtime); + tor_free(ent->filename); + tor_free(ent); + tor_free(expected_filename); + + /* + * Reset the mocks and check their state + */ + mock_unlink_reset(); + tt_int_op(unlinked_count, ==, 0); + reset_read_file_to_str_mock(); + tt_int_op(read_count, ==, 0); + + done: + + UNMOCK(tor_unlink); + mock_unlink_reset(); + UNMOCK(read_file_to_str); + reset_read_file_to_str_mock(); + + tor_free(file_content); + + return; +} + +static smartlist_t * +listdir_mock(const char *dname) +{ + smartlist_t *l; + + /* Ignore the name, always return this list */ + (void)dname; + + l = smartlist_new(); + smartlist_add(l, tor_strdup("foo")); + smartlist_add(l, tor_strdup("bar")); + smartlist_add(l, tor_strdup("baz")); + + return l; +} + +static dumped_desc_t * +pop_one_mock(const char *dirname, const char *f) +{ + dumped_desc_t *ent = NULL; + + if (dirname != NULL && strcmp(dirname, "d") == 0) { + if (f != NULL && strcmp(f, "foo") == 0) { + ent = tor_malloc_zero(sizeof(*ent)); + ent->filename = tor_strdup("d/foo"); + ent->len = 123; + ent->digest_sha256[0] = 1; + ent->when = 1024; + } else if (f != NULL && strcmp(f, "bar") == 0) { + ent = tor_malloc_zero(sizeof(*ent)); + ent->filename = tor_strdup("d/bar"); + ent->len = 456; + ent->digest_sha256[0] = 2; + /* + * Note that the timestamps are in a different order than + * listdir_mock() returns; we're testing the sort order. + */ + ent->when = 512; + } else if (f != NULL && strcmp(f, "baz") == 0) { + ent = tor_malloc_zero(sizeof(*ent)); + ent->filename = tor_strdup("d/baz"); + ent->len = 789; + ent->digest_sha256[0] = 3; + ent->when = 768; + } + } + + return ent; +} + +/* This one tests dump_desc_populate_fifo_from_directory() */ +static void +test_dir_populate_dump_desc_fifo_2(void *data) +{ + dumped_desc_t *ent = NULL; + + (void)data; + + /* Set up the mocks */ + MOCK(tor_listdir, listdir_mock); + MOCK(dump_desc_populate_one_file, pop_one_mock); + + /* Run dump_desc_populate_fifo_from_directory() */ + descs_dumped = NULL; + len_descs_dumped = 0; + dump_desc_populate_fifo_from_directory("d"); + tt_assert(descs_dumped != NULL); + tt_int_op(smartlist_len(descs_dumped), OP_EQ, 3); + tt_u64_op(len_descs_dumped, OP_EQ, 1368); + ent = smartlist_get(descs_dumped, 0); + tt_str_op(ent->filename, OP_EQ, "d/bar"); + tt_int_op(ent->len, OP_EQ, 456); + tt_int_op(ent->when, OP_EQ, 512); + ent = smartlist_get(descs_dumped, 1); + tt_str_op(ent->filename, OP_EQ, "d/baz"); + tt_int_op(ent->len, OP_EQ, 789); + tt_int_op(ent->when, OP_EQ, 768); + ent = smartlist_get(descs_dumped, 2); + tt_str_op(ent->filename, OP_EQ, "d/foo"); + tt_int_op(ent->len, OP_EQ, 123); + tt_int_op(ent->when, OP_EQ, 1024); + + done: + dump_desc_fifo_cleanup(); + + UNMOCK(dump_desc_populate_one_file); + UNMOCK(tor_listdir); + + return; +} + static int mock_networkstatus_consensus_is_bootstrapping_value = 0; static int mock_networkstatus_consensus_is_bootstrapping(time_t now) @@ -4093,7 +5232,7 @@ test_dir_find_dl_schedule(void* data) smartlist_t client_boot_auth_only_cons, client_boot_auth_cons; smartlist_t client_boot_fallback_cons, bridge; - mock_options = malloc(sizeof(or_options_t)); + mock_options = tor_malloc(sizeof(or_options_t)); reset_options(mock_options, &mock_get_options_calls); MOCK(get_options, mock_get_options); @@ -4202,7 +5341,7 @@ test_dir_find_dl_schedule(void* data) UNMOCK(networkstatus_consensus_is_bootstrapping); UNMOCK(networkstatus_consensus_can_use_extra_fallbacks); UNMOCK(get_options); - free(mock_options); + tor_free(mock_options); mock_options = NULL; } @@ -4230,6 +5369,7 @@ struct testcase_t dir_tests[] = { DIR_LEGACY(measured_bw_kb), DIR_LEGACY(measured_bw_kb_cache), DIR_LEGACY(param_voting), + DIR(param_voting_lookup, 0), DIR_LEGACY(v3_networkstatus), DIR(random_weighted, 0), DIR(scale_bw, 0), @@ -4242,6 +5382,7 @@ struct testcase_t dir_tests[] = { DIR(fetch_type, 0), DIR(packages, 0), DIR(download_status_schedule, 0), + DIR(download_status_random_backoff, 0), DIR(download_status_increment, 0), DIR(authdir_type_to_string, 0), DIR(conn_purpose_to_string, 0), @@ -4250,6 +5391,9 @@ struct testcase_t dir_tests[] = { DIR(should_not_init_request_to_dir_auths_without_v3_info, 0), DIR(should_init_request_to_dir_auths, 0), DIR(choose_compression_level, 0), + DIR(dump_unparseable_descriptors, 0), + DIR(populate_dump_desc_fifo, 0), + DIR(populate_dump_desc_fifo_2, 0), DIR_ARG(find_dl_schedule, TT_FORK, "bf"), DIR_ARG(find_dl_schedule, TT_FORK, "ba"), DIR_ARG(find_dl_schedule, TT_FORK, "cf"), diff --git a/src/test/test_dir_common.c b/src/test/test_dir_common.c index 0b446c2dfd..2448d307b2 100644 --- a/src/test/test_dir_common.c +++ b/src/test/test_dir_common.c @@ -21,13 +21,6 @@ networkstatus_t * dir_common_add_rs_and_parse(networkstatus_t *vote, crypto_pk_t *sign_skey, int *n_vrs, time_t now, int clear_rl); -extern const char AUTHORITY_CERT_1[]; -extern const char AUTHORITY_SIGNKEY_1[]; -extern const char AUTHORITY_CERT_2[]; -extern const char AUTHORITY_SIGNKEY_2[]; -extern const char AUTHORITY_CERT_3[]; -extern const char AUTHORITY_SIGNKEY_3[]; - /** Initialize and set auth certs and keys * Returns 0 on success, -1 on failure. Clean up handled by caller. */ diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index 05657ca452..3282037243 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -38,7 +38,15 @@ #include <dirent.h> #endif +#ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS +DISABLE_GCC_WARNING(overlength-strings) +/* We allow huge string constants in the unit tests, but not in the code + * at large. */ +#endif #include "vote_descriptors.inc" +#ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS +ENABLE_GCC_WARNING(overlength-strings) +#endif #define NS_MODULE dir_handle_get @@ -224,51 +232,6 @@ test_dir_handle_get_robots_txt(void *data) tor_free(body); } -static void -test_dir_handle_get_bytes_txt(void *data) -{ - dir_connection_t *conn = NULL; - char *header = NULL; - char *body = NULL; - size_t body_used = 0, body_len = 0; - char buff[30]; - char *exp_body = NULL; - (void) data; - - exp_body = directory_dump_request_log(); - body_len = strlen(exp_body); - - MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock); - - conn = dir_connection_new(tor_addr_family(&MOCK_TOR_ADDR)); - - tt_int_op(directory_handle_command_get(conn, GET("/tor/bytes.txt"), NULL, 0), - OP_EQ, 0); - fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE, - &body, &body_used, body_len+1, 0); - - tt_assert(header); - tt_assert(body); - - tt_ptr_op(strstr(header, "HTTP/1.0 200 OK\r\n"), OP_EQ, header); - tt_assert(strstr(header, "Content-Type: text/plain\r\n")); - tt_assert(strstr(header, "Content-Encoding: identity\r\n")); - tt_assert(strstr(header, "Pragma: no-cache\r\n")); - - tor_snprintf(buff, sizeof(buff), "Content-Length: %ld\r\n", (long) body_len); - tt_assert(strstr(header, buff)); - - tt_int_op(body_used, OP_EQ, strlen(body)); - tt_str_op(body, OP_EQ, exp_body); - - done: - UNMOCK(connection_write_to_buf_impl_); - connection_free_(TO_CONN(conn)); - tor_free(header); - tor_free(body); - tor_free(exp_body); -} - #define RENDEZVOUS2_GET(descid) GET("/tor/rendezvous2/" descid) static void test_dir_handle_get_rendezvous2_not_found_if_not_encrypted(void *data) @@ -438,7 +401,7 @@ test_dir_handle_get_rendezvous2_on_encrypted_conn_success(void *data) TO_CONN(conn)->linked = 1; tt_assert(connection_dir_is_encrypted(conn)); - sprintf(req, RENDEZVOUS2_GET("%s"), desc_id_base32); + tor_snprintf(req, sizeof(req), RENDEZVOUS2_GET("%s"), desc_id_base32); tt_int_op(directory_handle_command_get(conn, req, NULL, 0), OP_EQ, 0); @@ -453,7 +416,7 @@ test_dir_handle_get_rendezvous2_on_encrypted_conn_success(void *data) tt_assert(strstr(header, "Content-Type: text/plain\r\n")); tt_assert(strstr(header, "Content-Encoding: identity\r\n")); tt_assert(strstr(header, "Pragma: no-cache\r\n")); - sprintf(buff, "Content-Length: %ld\r\n", (long) body_len); + tor_snprintf(buff, sizeof(buff), "Content-Length: %ld\r\n", (long) body_len); tt_assert(strstr(header, buff)); tt_int_op(body_used, OP_EQ, strlen(body)); @@ -504,7 +467,7 @@ static or_options_t *mock_options = NULL; static void init_mock_options(void) { - mock_options = malloc(sizeof(or_options_t)); + mock_options = tor_malloc(sizeof(or_options_t)); memset(mock_options, 0, sizeof(or_options_t)); mock_options->TestingTorNetwork = 1; } @@ -565,7 +528,7 @@ test_dir_handle_get_micro_d(void *data) /* Make the request */ conn = dir_connection_new(tor_addr_family(&MOCK_TOR_ADDR)); - sprintf(path, MICRODESC_GET("%s"), digest_base64); + tor_snprintf(path, sizeof(path), MICRODESC_GET("%s"), digest_base64); tt_int_op(directory_handle_command_get(conn, path, NULL, 0), OP_EQ, 0); fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE, @@ -635,7 +598,7 @@ test_dir_handle_get_micro_d_server_busy(void *data) /* Make the request */ conn = dir_connection_new(tor_addr_family(&MOCK_TOR_ADDR)); - sprintf(path, MICRODESC_GET("%s"), digest_base64); + tor_snprintf(path, sizeof(path), MICRODESC_GET("%s"), digest_base64); tt_int_op(directory_handle_command_get(conn, path, NULL, 0), OP_EQ, 0); fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE, @@ -997,7 +960,8 @@ test_dir_handle_get_server_descriptors_fp(void* data) DIGEST_LEN); char req[155]; - sprintf(req, SERVER_DESC_GET("fp/%s+" HEX1 "+" HEX2), hex_digest); + tor_snprintf(req, sizeof(req), SERVER_DESC_GET("fp/%s+" HEX1 "+" HEX2), + hex_digest); tt_int_op(directory_handle_command_get(conn, req, NULL, 0), OP_EQ, 0); //TODO: Is this a BUG? @@ -1056,8 +1020,9 @@ test_dir_handle_get_server_descriptors_d(void* data) conn = dir_connection_new(tor_addr_family(&MOCK_TOR_ADDR)); - char req_header[155]; - sprintf(req_header, SERVER_DESC_GET("d/%s+" HEX1 "+" HEX2), hex_digest); + char req_header[155]; /* XXX Why 155? What kind of number is that?? */ + tor_snprintf(req_header, sizeof(req_header), + SERVER_DESC_GET("d/%s+" HEX1 "+" HEX2), hex_digest); tt_int_op(directory_handle_command_get(conn, req_header, NULL, 0), OP_EQ, 0); //TODO: Is this a BUG? @@ -1125,8 +1090,9 @@ test_dir_handle_get_server_descriptors_busy(void* data) #define HEX1 "Fe0daff89127389bc67558691231234551193EEE" #define HEX2 "Deadbeef99999991111119999911111111f00ba4" - char req_header[155]; - sprintf(req_header, SERVER_DESC_GET("d/%s+" HEX1 "+" HEX2), hex_digest); + char req_header[155]; /* XXX 155? Why 155? */ + tor_snprintf(req_header, sizeof(req_header), + SERVER_DESC_GET("d/%s+" HEX1 "+" HEX2), hex_digest); tt_int_op(directory_handle_command_get(conn, req_header, NULL, 0), OP_EQ, 0); fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE, @@ -1204,8 +1170,6 @@ test_dir_handle_get_server_keys_all_not_found(void* data) #define TEST_CERTIFICATE AUTHORITY_CERT_3 #define TEST_SIGNING_KEY AUTHORITY_SIGNKEY_A_DIGEST -extern const char AUTHORITY_CERT_3[]; -extern const char AUTHORITY_SIGNKEY_A_DIGEST[]; static const char TEST_CERT_IDENT_KEY[] = "D867ACF56A9D229B35C25F0090BC9867E906BE69"; @@ -1237,7 +1201,7 @@ test_dir_handle_get_server_keys_all(void* data) base16_decode(ds->v3_identity_digest, DIGEST_LEN, TEST_CERT_IDENT_KEY, HEX_DIGEST_LEN); tt_int_op(0, OP_EQ, trusted_dirs_load_certs_from_string(TEST_CERTIFICATE, - TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST, 1)); + TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST, 1, NULL)); conn = dir_connection_new(tor_addr_family(&MOCK_TOR_ADDR)); @@ -1396,11 +1360,12 @@ test_dir_handle_get_server_keys_fp(void* data) TEST_CERT_IDENT_KEY, HEX_DIGEST_LEN); tt_int_op(0, OP_EQ, trusted_dirs_load_certs_from_string(TEST_CERTIFICATE, - TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST, 1)); + TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST, 1, NULL)); conn = dir_connection_new(tor_addr_family(&MOCK_TOR_ADDR)); char req[71]; - sprintf(req, GET("/tor/keys/fp/%s"), TEST_CERT_IDENT_KEY); + tor_snprintf(req, sizeof(req), + GET("/tor/keys/fp/%s"), TEST_CERT_IDENT_KEY); tt_int_op(directory_handle_command_get(conn, req, NULL, 0), OP_EQ, 0); fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE, @@ -1468,11 +1433,12 @@ test_dir_handle_get_server_keys_sk(void* data) routerlist_free_all(); tt_int_op(0, OP_EQ, trusted_dirs_load_certs_from_string(TEST_CERTIFICATE, - TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST, 1)); + TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST, 1, NULL)); conn = dir_connection_new(tor_addr_family(&MOCK_TOR_ADDR)); char req[71]; - sprintf(req, GET("/tor/keys/sk/%s"), TEST_SIGNING_KEY); + tor_snprintf(req, sizeof(req), + GET("/tor/keys/sk/%s"), TEST_SIGNING_KEY); tt_int_op(directory_handle_command_get(conn, req, NULL, 0), OP_EQ, 0); fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE, @@ -1550,13 +1516,14 @@ test_dir_handle_get_server_keys_fpsk(void* data) dir_server_add(ds); tt_int_op(0, OP_EQ, trusted_dirs_load_certs_from_string(TEST_CERTIFICATE, - TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST, 1)); + TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST, 1, NULL)); conn = dir_connection_new(tor_addr_family(&MOCK_TOR_ADDR)); char req[115]; - sprintf(req, GET("/tor/keys/fp-sk/%s-%s"), - TEST_CERT_IDENT_KEY, TEST_SIGNING_KEY); + tor_snprintf(req, sizeof(req), + GET("/tor/keys/fp-sk/%s-%s"), + TEST_CERT_IDENT_KEY, TEST_SIGNING_KEY); tt_int_op(directory_handle_command_get(conn, req, NULL, 0), OP_EQ, 0); @@ -1606,7 +1573,7 @@ test_dir_handle_get_server_keys_busy(void* data) dir_server_add(ds); tt_int_op(0, OP_EQ, trusted_dirs_load_certs_from_string(TEST_CERTIFICATE, - TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST, 1)); + TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST, 1, NULL)); MOCK(get_options, mock_get_options); MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock); @@ -1617,7 +1584,7 @@ test_dir_handle_get_server_keys_busy(void* data) conn = dir_connection_new(tor_addr_family(&MOCK_TOR_ADDR)); char req[71]; - sprintf(req, GET("/tor/keys/fp/%s"), TEST_CERT_IDENT_KEY); + tor_snprintf(req, sizeof(req), GET("/tor/keys/fp/%s"), TEST_CERT_IDENT_KEY); tt_int_op(directory_handle_command_get(conn, req, NULL, 0), OP_EQ, 0); fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE, @@ -2344,7 +2311,7 @@ test_dir_handle_get_status_vote_next_authority(void* data) base16_decode(ds->v3_identity_digest, DIGEST_LEN, TEST_CERT_IDENT_KEY, HEX_DIGEST_LEN); tt_int_op(0, OP_EQ, trusted_dirs_load_certs_from_string(TEST_CERTIFICATE, - TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST, 1)); + TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST, 1, NULL)); init_mock_options(); mock_options->AuthoritativeDir = 1; @@ -2423,7 +2390,7 @@ test_dir_handle_get_status_vote_current_authority(void* data) TEST_CERT_IDENT_KEY, HEX_DIGEST_LEN); tt_int_op(0, OP_EQ, trusted_dirs_load_certs_from_string(TEST_CERTIFICATE, - TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST, 1)); + TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST, 1, NULL)); init_mock_options(); mock_options->AuthoritativeDir = 1; @@ -2484,7 +2451,6 @@ struct testcase_t dir_handle_get_tests[] = { DIR_HANDLE_CMD(v1_command_not_found, 0), DIR_HANDLE_CMD(v1_command, 0), DIR_HANDLE_CMD(robots_txt, 0), - DIR_HANDLE_CMD(bytes_txt, 0), DIR_HANDLE_CMD(rendezvous2_not_found_if_not_encrypted, 0), DIR_HANDLE_CMD(rendezvous2_not_found, 0), DIR_HANDLE_CMD(rendezvous2_on_encrypted_conn_with_invalid_desc_id, 0), diff --git a/src/test/test_guardfraction.c b/src/test/test_guardfraction.c index 300590a3d9..130aff11aa 100644 --- a/src/test/test_guardfraction.c +++ b/src/test/test_guardfraction.c @@ -40,7 +40,7 @@ gen_vote_routerstatus_for_tests(const char *digest_in_hex, int is_guard) tt_int_op(strlen(digest_in_hex), ==, HEX_DIGEST_LEN); retval = base16_decode(digest_tmp, sizeof(digest_tmp), digest_in_hex, HEX_DIGEST_LEN); - tt_int_op(retval, ==, 0); + tt_int_op(retval, ==, sizeof(digest_tmp)); memcpy(rs->identity_digest, digest_tmp, DIGEST_LEN); } diff --git a/src/test/test_handles.c b/src/test/test_handles.c new file mode 100644 index 0000000000..536a478689 --- /dev/null +++ b/src/test/test_handles.c @@ -0,0 +1,95 @@ +/* Copyright (c) 2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include "test.h" + +#include "util.h" +#include "handles.h" + +typedef struct demo_t { + HANDLE_ENTRY(demo, demo_t); + int val; +} demo_t; + +HANDLE_DECL(demo, demo_t, static) +HANDLE_IMPL(demo, demo_t, static) + +static demo_t * +demo_new(int val) +{ + demo_t *d = tor_malloc_zero(sizeof(demo_t)); + d->val = val; + return d; +} + +static void +demo_free(demo_t *d) +{ + if (d == NULL) + return; + demo_handles_clear(d); + tor_free(d); +} + +static void +test_handle_basic(void *arg) +{ + (void) arg; + demo_t *d1 = NULL, *d2 = NULL; + demo_handle_t *wr1 = NULL, *wr2 = NULL, *wr3 = NULL, *wr4 = NULL; + + d1 = demo_new(9000); + d2 = demo_new(9009); + + wr1 = demo_handle_new(d1); + wr2 = demo_handle_new(d1); + wr3 = demo_handle_new(d1); + wr4 = demo_handle_new(d2); + + tt_assert(wr1); + tt_assert(wr2); + tt_assert(wr3); + tt_assert(wr4); + + tt_ptr_op(demo_handle_get(wr1), OP_EQ, d1); + tt_ptr_op(demo_handle_get(wr2), OP_EQ, d1); + tt_ptr_op(demo_handle_get(wr3), OP_EQ, d1); + tt_ptr_op(demo_handle_get(wr4), OP_EQ, d2); + + demo_handle_free(wr1); + wr1 = NULL; + tt_ptr_op(demo_handle_get(wr2), OP_EQ, d1); + tt_ptr_op(demo_handle_get(wr3), OP_EQ, d1); + tt_ptr_op(demo_handle_get(wr4), OP_EQ, d2); + + demo_free(d1); + d1 = NULL; + tt_ptr_op(demo_handle_get(wr2), OP_EQ, NULL); + tt_ptr_op(demo_handle_get(wr3), OP_EQ, NULL); + tt_ptr_op(demo_handle_get(wr4), OP_EQ, d2); + + demo_handle_free(wr2); + wr2 = NULL; + tt_ptr_op(demo_handle_get(wr3), OP_EQ, NULL); + tt_ptr_op(demo_handle_get(wr4), OP_EQ, d2); + + demo_handle_free(wr3); + wr3 = NULL; + done: + demo_handle_free(wr1); + demo_handle_free(wr2); + demo_handle_free(wr3); + demo_handle_free(wr4); + demo_free(d1); + demo_free(d2); +} + +#define HANDLE_TEST(name, flags) \ + { #name, test_handle_ ##name, (flags), NULL, NULL } + +struct testcase_t handle_tests[] = { + HANDLE_TEST(basic, 0), + END_OF_TESTCASES +}; + diff --git a/src/test/test_helpers.c b/src/test/test_helpers.c index c6daaf220a..ae9fc7a243 100644 --- a/src/test/test_helpers.c +++ b/src/test/test_helpers.c @@ -16,7 +16,15 @@ #include "test.h" #include "test_helpers.h" +#ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS +DISABLE_GCC_WARNING(overlength-strings) +/* We allow huge string constants in the unit tests, but not in the code + * at large. */ +#endif #include "test_descriptors.inc" +#ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS +ENABLE_GCC_WARNING(overlength-strings) +#endif /* Return a statically allocated string representing yesterday's date * in ISO format. We use it so that state file items are not found to diff --git a/src/test/test_hs.c b/src/test/test_hs.c index 49939a53cf..1daa1552e9 100644 --- a/src/test/test_hs.c +++ b/src/test/test_hs.c @@ -435,6 +435,67 @@ test_hs_rend_data(void *arg) rend_data_free(client_dup); } +/* Test encoding and decoding service authorization cookies */ +static void +test_hs_auth_cookies(void *arg) +{ +#define TEST_COOKIE_RAW ((const uint8_t *) "abcdefghijklmnop") +#define TEST_COOKIE_ENCODED "YWJjZGVmZ2hpamtsbW5vcA" +#define TEST_COOKIE_ENCODED_STEALTH "YWJjZGVmZ2hpamtsbW5vcB" +#define TEST_COOKIE_ENCODED_INVALID "YWJjZGVmZ2hpamtsbW5vcD" + + char *encoded_cookie; + uint8_t raw_cookie[REND_DESC_COOKIE_LEN]; + rend_auth_type_t auth_type; + char *err_msg; + int re; + + (void)arg; + + /* Test that encoding gives the expected result */ + encoded_cookie = rend_auth_encode_cookie(TEST_COOKIE_RAW, REND_BASIC_AUTH); + tt_str_op(encoded_cookie, OP_EQ, TEST_COOKIE_ENCODED); + tor_free(encoded_cookie); + + encoded_cookie = rend_auth_encode_cookie(TEST_COOKIE_RAW, REND_STEALTH_AUTH); + tt_str_op(encoded_cookie, OP_EQ, TEST_COOKIE_ENCODED_STEALTH); + tor_free(encoded_cookie); + + /* Decoding should give the original value */ + re = rend_auth_decode_cookie(TEST_COOKIE_ENCODED, raw_cookie, &auth_type, + &err_msg); + tt_assert(!re); + tt_assert(!err_msg); + tt_mem_op(raw_cookie, OP_EQ, TEST_COOKIE_RAW, REND_DESC_COOKIE_LEN); + tt_int_op(auth_type, OP_EQ, REND_BASIC_AUTH); + memset(raw_cookie, 0, sizeof(raw_cookie)); + + re = rend_auth_decode_cookie(TEST_COOKIE_ENCODED_STEALTH, raw_cookie, + &auth_type, &err_msg); + tt_assert(!re); + tt_assert(!err_msg); + tt_mem_op(raw_cookie, OP_EQ, TEST_COOKIE_RAW, REND_DESC_COOKIE_LEN); + tt_int_op(auth_type, OP_EQ, REND_STEALTH_AUTH); + memset(raw_cookie, 0, sizeof(raw_cookie)); + + /* Decoding with padding characters should also work */ + re = rend_auth_decode_cookie(TEST_COOKIE_ENCODED "==", raw_cookie, NULL, + &err_msg); + tt_assert(!re); + tt_assert(!err_msg); + tt_mem_op(raw_cookie, OP_EQ, TEST_COOKIE_RAW, REND_DESC_COOKIE_LEN); + + /* Decoding with an unknown type should fail */ + re = rend_auth_decode_cookie(TEST_COOKIE_ENCODED_INVALID, raw_cookie, + &auth_type, &err_msg); + tt_int_op(re, OP_LT, 0); + tt_assert(err_msg); + tor_free(err_msg); + + done: + return; +} + struct testcase_t hs_tests[] = { { "hs_rend_data", test_hs_rend_data, TT_FORK, NULL, NULL }, @@ -445,6 +506,8 @@ struct testcase_t hs_tests[] = { { "pick_bad_tor2web_rendezvous_node", test_pick_bad_tor2web_rendezvous_node, TT_FORK, NULL, NULL }, + { "hs_auth_cookies", test_hs_auth_cookies, TT_FORK, + NULL, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_introduce.c b/src/test/test_introduce.c index 9c7a86da66..810b03c93d 100644 --- a/src/test/test_introduce.c +++ b/src/test/test_introduce.c @@ -9,8 +9,6 @@ #define RENDSERVICE_PRIVATE #include "rendservice.h" -extern const char AUTHORITY_SIGNKEY_1[]; - static uint8_t v0_test_plaintext[] = /* 20 bytes of rendezvous point nickname */ { 0x4e, 0x69, 0x63, 0x6b, 0x6e, 0x61, 0x6d, 0x65, diff --git a/src/test/test_link_handshake.c b/src/test/test_link_handshake.c index e8856c60de..4038783459 100644 --- a/src/test/test_link_handshake.c +++ b/src/test/test_link_handshake.c @@ -16,7 +16,7 @@ #include "test.h" -var_cell_t *mock_got_var_cell = NULL; +static var_cell_t *mock_got_var_cell = NULL; static void mock_write_var_cell(const var_cell_t *vc, or_connection_t *conn) diff --git a/src/test/test_logging.c b/src/test/test_logging.c index eb294fe6f8..15471e46d0 100644 --- a/src/test/test_logging.c +++ b/src/test/test_logging.c @@ -127,9 +127,47 @@ test_sigsafe_err(void *arg) smartlist_free(lines); } +static void +test_ratelim(void *arg) +{ + (void) arg; + ratelim_t ten_min = RATELIM_INIT(10*60); + + const time_t start = 1466091600; + time_t now = start; + /* Initially, we're ready. */ + + char *msg = NULL; + + msg = rate_limit_log(&ten_min, now); + tt_assert(msg != NULL); + tt_str_op(msg, OP_EQ, ""); /* nothing was suppressed. */ + + tt_int_op(ten_min.last_allowed, OP_EQ, now); + tor_free(msg); + + int i; + for (i = 0; i < 9; ++i) { + now += 60; /* one minute has passed. */ + msg = rate_limit_log(&ten_min, now); + tt_assert(msg == NULL); + tt_int_op(ten_min.last_allowed, OP_EQ, start); + tt_int_op(ten_min.n_calls_since_last_time, OP_EQ, i + 1); + } + + now += 240; /* Okay, we can be done. */ + msg = rate_limit_log(&ten_min, now); + tt_assert(msg != NULL); + tt_str_op(msg, OP_EQ, + " [9 similar message(s) suppressed in last 600 seconds]"); + done: + tor_free(msg); +} + struct testcase_t logging_tests[] = { { "sigsafe_err_fds", test_get_sigsafe_err_fds, TT_FORK, NULL, NULL }, { "sigsafe_err", test_sigsafe_err, TT_FORK, NULL, NULL }, + { "ratelim", test_ratelim, 0, NULL, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_microdesc.c b/src/test/test_microdesc.c index dbd1e5ac48..2afbdde88a 100644 --- a/src/test/test_microdesc.c +++ b/src/test/test_microdesc.c @@ -14,30 +14,11 @@ #include "test.h" -#ifdef __GNUC__ -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -#endif - -#if __GNUC__ && GCC_VERSION >= 402 -#if GCC_VERSION >= 406 -#pragma GCC diagnostic push -#endif -/* Some versions of OpenSSL declare X509_STORE_CTX_set_verify_cb twice. - * Suppress the GCC warning so we can build with -Wredundant-decl. */ -#pragma GCC diagnostic ignored "-Wredundant-decls" -#endif - +DISABLE_GCC_WARNING(redundant-decls) #include <openssl/rsa.h> #include <openssl/bn.h> #include <openssl/pem.h> - -#if __GNUC__ && GCC_VERSION >= 402 -#if GCC_VERSION >= 406 -#pragma GCC diagnostic pop -#else -#pragma GCC diagnostic warning "-Wredundant-decls" -#endif -#endif +ENABLE_GCC_WARNING(redundant-decls) #ifdef _WIN32 /* For mkdir() */ @@ -511,6 +492,11 @@ test_md_generate(void *arg) routerinfo_free(ri); } +#ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS +DISABLE_GCC_WARNING(overlength-strings) +/* We allow huge string constants in the unit tests, but not in the code + * at large. */ +#endif /* Taken at random from my ~/.tor/cached-microdescs file and then * hand-munged */ static const char MD_PARSE_TEST_DATA[] = @@ -666,6 +652,9 @@ static const char MD_PARSE_TEST_DATA[] = "id rsa1024 2A8wYpHxnkKJ92orocvIQBzeHlE\n" "p6 allow 80\n" ; +#ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS +ENABLE_GCC_WARNING(overlength-strings) +#endif /** More tests for parsing different kinds of microdescriptors, and getting * invalid digests trackd from them. */ @@ -794,7 +783,8 @@ test_md_reject_cache(void *arg) mc = get_microdesc_cache(); #define ADD(hex) \ do { \ - tt_int_op(0,OP_EQ,base16_decode(buf,sizeof(buf),hex,strlen(hex))); \ + tt_int_op(sizeof(buf),OP_EQ,base16_decode(buf,sizeof(buf), \ + hex,strlen(hex)));\ smartlist_add(wanted, tor_memdup(buf, DIGEST256_LEN)); \ } while (0) diff --git a/src/test/test_ntor_cl.c b/src/test/test_ntor_cl.c index 6df123162e..a560e5fc5e 100644 --- a/src/test/test_ntor_cl.c +++ b/src/test/test_ntor_cl.c @@ -21,7 +21,7 @@ } STMT_END #define BASE16(idx, var, n) STMT_BEGIN { \ const char *s = argv[(idx)]; \ - if (base16_decode((char*)var, n, s, strlen(s)) < 0 ) { \ + if (base16_decode((char*)var, n, s, strlen(s)) < (int)n ) { \ fprintf(stderr, "couldn't decode argument %d (%s)\n",idx,s); \ return 1; \ } \ @@ -153,7 +153,10 @@ main(int argc, char **argv) if (argc < 2) { fprintf(stderr, "I need arguments. Read source for more info.\n"); return 1; - } else if (!strcmp(argv[1], "client1")) { + } + + curve25519_init(); + if (!strcmp(argv[1], "client1")) { return client1(argc, argv); } else if (!strcmp(argv[1], "server1")) { return server1(argc, argv); diff --git a/src/test/test_options.c b/src/test/test_options.c index 4f24757a85..8d1d6f901e 100644 --- a/src/test/test_options.c +++ b/src/test/test_options.c @@ -12,7 +12,7 @@ #define ROUTERSET_PRIVATE #include "routerset.h" - +#include "main.h" #include "log_test_helpers.h" #include "sandbox.h" @@ -513,8 +513,9 @@ test_options_validate__nickname(void *ignored) ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg); tt_int_op(ret, OP_EQ, -1); tt_str_op(msg, OP_EQ, - "Nickname 'ThisNickNameIsABitTooLong' is wrong length or" - " contains illegal characters."); + "Nickname 'ThisNickNameIsABitTooLong', nicknames must be between " + "1 and 19 characters inclusive, and must contain only the " + "characters [a-zA-Z0-9]."); tor_free(msg); free_options_test_data(tdata); @@ -571,8 +572,6 @@ test_options_validate__contactinfo(void *ignored) tor_free(msg); } -extern int quiet_level; - static void test_options_validate__logs(void *ignored) { diff --git a/src/test/test_policy.c b/src/test/test_policy.c index a939ebf54f..14182af86b 100644 --- a/src/test/test_policy.c +++ b/src/test/test_policy.c @@ -778,8 +778,8 @@ test_policies_reject_port_address(void *arg) UNMOCK(get_configured_ports); } -smartlist_t *mock_ipv4_addrs = NULL; -smartlist_t *mock_ipv6_addrs = NULL; +static smartlist_t *mock_ipv4_addrs = NULL; +static smartlist_t *mock_ipv6_addrs = NULL; /* mock get_interface_address6_list, returning a deep copy of the template * address list ipv4_interface_address_list or ipv6_interface_address_list */ @@ -804,7 +804,7 @@ mock_get_interface_address6_list(int severity, tt_assert(template_list); SMARTLIST_FOREACH_BEGIN(template_list, tor_addr_t *, src_addr) { - tor_addr_t *dest_addr = malloc(sizeof(tor_addr_t)); + tor_addr_t *dest_addr = tor_malloc(sizeof(tor_addr_t)); memset(dest_addr, 0, sizeof(*dest_addr)); tor_addr_copy_tight(dest_addr, src_addr); smartlist_add(clone_list, dest_addr); diff --git a/src/test/test_pubsub.c b/src/test/test_pubsub.c new file mode 100644 index 0000000000..547d6c6b32 --- /dev/null +++ b/src/test/test_pubsub.c @@ -0,0 +1,85 @@ +/* Copyright (c) 2016, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_pubsub.c + * \brief Unit tests for publish-subscribe abstraction. + **/ + +#include "or.h" +#include "test.h" +#include "pubsub.h" + +DECLARE_PUBSUB_STRUCT_TYPES(foobar) +DECLARE_PUBSUB_TOPIC(foobar) +DECLARE_NOTIFY_PUBSUB_TOPIC(static, foobar) +IMPLEMENT_PUBSUB_TOPIC(static, foobar) + +struct foobar_event_data_t { + unsigned u; + const char *s; +}; + +struct foobar_subscriber_data_t { + const char *name; + long l; +}; + +static int +foobar_sub1(foobar_event_data_t *ev, foobar_subscriber_data_t *mine) +{ + ev->u += 10; + mine->l += 100; + return 0; +} + +static int +foobar_sub2(foobar_event_data_t *ev, foobar_subscriber_data_t *mine) +{ + ev->u += 5; + mine->l += 50; + return 0; +} + +static void +test_pubsub_basic(void *arg) +{ + (void)arg; + foobar_subscriber_data_t subdata1 = { "hi", 0 }; + foobar_subscriber_data_t subdata2 = { "wow", 0 }; + const foobar_subscriber_t *sub1; + const foobar_subscriber_t *sub2; + foobar_event_data_t ed = { 0, "x" }; + foobar_event_data_t ed2 = { 0, "y" }; + sub1 = foobar_subscribe(foobar_sub1, &subdata1, SUBSCRIBE_ATSTART, 100); + tt_assert(sub1); + + foobar_notify(&ed, 0); + tt_int_op(subdata1.l, OP_EQ, 100); + tt_int_op(subdata2.l, OP_EQ, 0); + tt_int_op(ed.u, OP_EQ, 10); + + sub2 = foobar_subscribe(foobar_sub2, &subdata2, 0, 5); + tt_assert(sub2); + + foobar_notify(&ed2, 0); + tt_int_op(subdata1.l, OP_EQ, 200); + tt_int_op(subdata2.l, OP_EQ, 50); + tt_int_op(ed2.u, OP_EQ, 15); + + foobar_unsubscribe(sub1); + + foobar_notify(&ed, 0); + tt_int_op(subdata1.l, OP_EQ, 200); + tt_int_op(subdata2.l, OP_EQ, 100); + tt_int_op(ed.u, OP_EQ, 15); + + done: + foobar_clear(); +} + +struct testcase_t pubsub_tests[] = { + { "pubsub_basic", test_pubsub_basic, TT_FORK, NULL, NULL }, + END_OF_TESTCASES +}; + diff --git a/src/test/test_relaycell.c b/src/test/test_relaycell.c index 1cd9ff064b..fb6748965a 100644 --- a/src/test/test_relaycell.c +++ b/src/test/test_relaycell.c @@ -95,7 +95,7 @@ test_relaycell_resolved(void *arg) tt_int_op(srm_ncalls, OP_EQ, 1); \ tt_ptr_op(srm_conn, OP_EQ, entryconn); \ tt_int_op(srm_atype, OP_EQ, (atype)); \ - if (answer) { \ + if ((answer) != NULL) { \ tt_int_op(srm_alen, OP_EQ, sizeof(answer)-1); \ tt_int_op(srm_alen, OP_LT, 512); \ tt_int_op(srm_answer_is_set, OP_EQ, 1); \ diff --git a/src/test/test_rendcache.c b/src/test/test_rendcache.c index d1b52649b2..e210e053b6 100644 --- a/src/test/test_rendcache.c +++ b/src/test/test_rendcache.c @@ -17,13 +17,8 @@ static const int RECENT_TIME = -10; static const int TIME_IN_THE_PAST = -(REND_CACHE_MAX_AGE + \ - REND_CACHE_MAX_SKEW + 10); -static const int TIME_IN_THE_FUTURE = REND_CACHE_MAX_SKEW + 10; - -extern strmap_t *rend_cache; -extern digestmap_t *rend_cache_v2_dir; -extern strmap_t *rend_cache_failure; -extern size_t rend_cache_total_allocation; + REND_CACHE_MAX_SKEW + 60); +static const int TIME_IN_THE_FUTURE = REND_CACHE_MAX_SKEW + 60; static rend_data_t * mock_rend_data(const char *onion_address) @@ -976,7 +971,7 @@ test_rend_cache_entry_free(void *data) // Handles non-NULL descriptor correctly e = tor_malloc_zero(sizeof(rend_cache_entry_t)); - e->desc = (char *)malloc(10); + e->desc = tor_malloc(10); rend_cache_entry_free(e); /* done: */ diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c index 2cffa6e801..088bd257c3 100644 --- a/src/test/test_routerlist.c +++ b/src/test/test_routerlist.c @@ -19,20 +19,24 @@ #include "networkstatus.h" #include "nodelist.h" #include "policies.h" +#include "router.h" #include "routerlist.h" #include "routerparse.h" +#include "shared_random.h" #include "test.h" #include "test_dir_common.h" -extern const char AUTHORITY_CERT_1[]; -extern const char AUTHORITY_SIGNKEY_1[]; -extern const char AUTHORITY_CERT_2[]; -extern const char AUTHORITY_SIGNKEY_2[]; -extern const char AUTHORITY_CERT_3[]; -extern const char AUTHORITY_SIGNKEY_3[]; - void construct_consensus(char **consensus_text_md); +static authority_cert_t *mock_cert; + +static authority_cert_t * +get_my_v3_authority_cert_m(void) +{ + tor_assert(mock_cert); + return mock_cert; +} + /* 4 digests + 3 sep + pre + post + NULL */ static char output[4*BASE64_DIGEST256_LEN+3+2+2+1]; @@ -234,6 +238,12 @@ test_router_pick_directory_server_impl(void *arg) tt_assert(networkstatus_consensus_is_bootstrapping(now + 2*24*60*60)); tt_assert(networkstatus_consensus_is_bootstrapping(now - 2*24*60*60)); + /* Init SR subsystem. */ + MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); + mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + sr_init(0); + UNMOCK(get_my_v3_authority_cert); + /* No consensus available, fail early */ rs = router_pick_directory_server_impl(V3_DIRINFO, (const int) 0, NULL); tt_assert(rs == NULL); @@ -423,7 +433,7 @@ test_router_pick_directory_server_impl(void *arg) networkstatus_vote_free(con_md); } -connection_t *mocked_connection = NULL; +static connection_t *mocked_connection = NULL; /* Mock connection_get_by_type_addr_port_purpose by returning * mocked_connection. */ diff --git a/src/test/test_routerset.c b/src/test/test_routerset.c index 74b39c0486..1b526d430b 100644 --- a/src/test/test_routerset.c +++ b/src/test/test_routerset.c @@ -432,7 +432,7 @@ NS(test_main)(void *arg) NS_DECL(addr_policy_t *, router_parse_addr_policy_item_from_string, (const char *s, int assume_action, int *malformed_list)); -addr_policy_t *NS(mock_addr_policy); +static addr_policy_t *NS(mock_addr_policy); static void NS(test_main)(void *arg) @@ -480,7 +480,7 @@ NS(router_parse_addr_policy_item_from_string)(const char *s, NS_DECL(addr_policy_t *, router_parse_addr_policy_item_from_string, (const char *s, int assume_action, int *bogus)); -addr_policy_t *NS(mock_addr_policy); +static addr_policy_t *NS(mock_addr_policy); static void NS(test_main)(void *arg) @@ -527,7 +527,7 @@ NS(router_parse_addr_policy_item_from_string)(const char *s, int assume_action, NS_DECL(addr_policy_t *, router_parse_addr_policy_item_from_string, (const char *s, int assume_action, int *bad)); -addr_policy_t *NS(mock_addr_policy); +static addr_policy_t *NS(mock_addr_policy); static void NS(test_main)(void *arg) @@ -1477,7 +1477,7 @@ NS(test_main)(void *arg) * routerset or routerinfo. */ -node_t NS(mock_node); +static node_t NS(mock_node); static void NS(test_main)(void *arg) @@ -1504,7 +1504,7 @@ NS(test_main)(void *arg) * routerset and no routerinfo. */ -node_t NS(mock_node); +static node_t NS(mock_node); static void NS(test_main)(void *arg) @@ -1603,7 +1603,7 @@ NS(test_main)(void *arg) NS_DECL(const node_t *, node_get_by_nickname, (const char *nickname, int warn_if_unused)); -const char *NS(mock_nickname); +static const char *NS(mock_nickname); static void NS(test_main)(void *arg) @@ -1652,8 +1652,8 @@ NS(node_get_by_nickname)(const char *nickname, int warn_if_unused) NS_DECL(const node_t *, node_get_by_nickname, (const char *nickname, int warn_if_unused)); -const char *NS(mock_nickname); -node_t NS(mock_node); +static const char *NS(mock_nickname); +static node_t NS(mock_node); static void NS(test_main)(void *arg) @@ -1702,8 +1702,8 @@ NS(node_get_by_nickname)(const char *nickname, int warn_if_unused) NS_DECL(const node_t *, node_get_by_nickname, (const char *nickname, int warn_if_unused)); -char *NS(mock_nickname); -node_t NS(mock_node); +static char *NS(mock_nickname); +static node_t NS(mock_node); static void NS(test_main)(void *arg) @@ -1754,7 +1754,7 @@ NS(node_get_by_nickname)(const char *nickname, int warn_if_unused) NS_DECL(smartlist_t *, nodelist_get_list, (void)); -smartlist_t *NS(mock_smartlist); +static smartlist_t *NS(mock_smartlist); static void NS(test_main)(void *arg) @@ -1800,8 +1800,8 @@ NS(nodelist_get_list)(void) NS_DECL(smartlist_t *, nodelist_get_list, (void)); -smartlist_t *NS(mock_smartlist); -node_t NS(mock_node); +static smartlist_t *NS(mock_smartlist); +static node_t NS(mock_node); static void NS(test_main)(void *arg) diff --git a/src/test/test_scheduler.c b/src/test/test_scheduler.c index 6e9889b48b..15fbb2d186 100644 --- a/src/test/test_scheduler.c +++ b/src/test/test_scheduler.c @@ -24,12 +24,6 @@ #include "test.h" #include "fakechans.h" -/* Statics in scheduler.c exposed to the test suite */ -extern smartlist_t *channels_pending; -extern struct event *run_sched_ev; -extern uint64_t queue_heuristic; -extern time_t queue_heuristic_timestamp; - /* Event base for scheduelr tests */ static struct event_base *mock_event_base = NULL; diff --git a/src/test/test_shared_random.c b/src/test/test_shared_random.c new file mode 100644 index 0000000000..886cee3a33 --- /dev/null +++ b/src/test/test_shared_random.c @@ -0,0 +1,1280 @@ +#define SHARED_RANDOM_PRIVATE +#define SHARED_RANDOM_STATE_PRIVATE +#define CONFIG_PRIVATE +#define DIRVOTE_PRIVATE + +#include "or.h" +#include "test.h" +#include "config.h" +#include "dirvote.h" +#include "shared_random.h" +#include "shared_random_state.h" +#include "routerkeys.h" +#include "routerlist.h" +#include "router.h" +#include "routerparse.h" +#include "networkstatus.h" + +static authority_cert_t *mock_cert; + +static authority_cert_t * +get_my_v3_authority_cert_m(void) +{ + tor_assert(mock_cert); + return mock_cert; +} + +static dir_server_t ds; + +static dir_server_t * +trusteddirserver_get_by_v3_auth_digest_m(const char *digest) +{ + (void) digest; + /* The shared random code only need to know if a valid pointer to a dir + * server object has been found so this is safe because it won't use the + * pointer at all never. */ + return &ds; +} + +/* Setup a minimal dirauth environment by initializing the SR state and + * making sure the options are set to be an authority directory. */ +static void +init_authority_state(void) +{ + MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); + + or_options_t *options = get_options_mutable(); + mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + tt_assert(mock_cert); + options->AuthoritativeDir = 1; + tt_int_op(0, ==, load_ed_keys(options, time(NULL))); + sr_state_init(0, 0); + /* It's possible a commit has been generated in our state depending on + * the phase we are currently in which uses "now" as the starting + * timestamp. Delete it before we do any testing below. */ + sr_state_delete_commits(); + + done: + UNMOCK(get_my_v3_authority_cert); +} + +static void +test_get_sr_protocol_phase(void *arg) +{ + time_t the_time; + sr_phase_t phase; + int retval; + + (void) arg; + + /* Initialize SR state */ + init_authority_state(); + + { + retval = parse_rfc1123_time("Wed, 20 Apr 2015 23:59:00 UTC", &the_time); + tt_int_op(retval, ==, 0); + + phase = get_sr_protocol_phase(the_time); + tt_int_op(phase, ==, SR_PHASE_REVEAL); + } + + { + retval = parse_rfc1123_time("Wed, 20 Apr 2015 00:00:00 UTC", &the_time); + tt_int_op(retval, ==, 0); + + phase = get_sr_protocol_phase(the_time); + tt_int_op(phase, ==, SR_PHASE_COMMIT); + } + + { + retval = parse_rfc1123_time("Wed, 20 Apr 2015 00:00:01 UTC", &the_time); + tt_int_op(retval, ==, 0); + + phase = get_sr_protocol_phase(the_time); + tt_int_op(phase, ==, SR_PHASE_COMMIT); + } + + { + retval = parse_rfc1123_time("Wed, 20 Apr 2015 11:59:00 UTC", &the_time); + tt_int_op(retval, ==, 0); + + phase = get_sr_protocol_phase(the_time); + tt_int_op(phase, ==, SR_PHASE_COMMIT); + } + + { + retval = parse_rfc1123_time("Wed, 20 Apr 2015 12:00:00 UTC", &the_time); + tt_int_op(retval, ==, 0); + + phase = get_sr_protocol_phase(the_time); + tt_int_op(phase, ==, SR_PHASE_REVEAL); + } + + { + retval = parse_rfc1123_time("Wed, 20 Apr 2015 12:00:01 UTC", &the_time); + tt_int_op(retval, ==, 0); + + phase = get_sr_protocol_phase(the_time); + tt_int_op(phase, ==, SR_PHASE_REVEAL); + } + + { + retval = parse_rfc1123_time("Wed, 20 Apr 2015 13:00:00 UTC", &the_time); + tt_int_op(retval, ==, 0); + + phase = get_sr_protocol_phase(the_time); + tt_int_op(phase, ==, SR_PHASE_REVEAL); + } + + done: + ; +} + +static networkstatus_t *mock_consensus = NULL; + +static void +test_get_state_valid_until_time(void *arg) +{ + time_t current_time; + time_t valid_until_time; + char tbuf[ISO_TIME_LEN + 1]; + int retval; + + (void) arg; + + { + /* Get the valid until time if called at 00:00:01 */ + retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:01 UTC", + ¤t_time); + tt_int_op(retval, ==, 0); + valid_until_time = get_state_valid_until_time(current_time); + + /* Compare it with the correct result */ + format_iso_time(tbuf, valid_until_time); + tt_str_op("2015-04-21 00:00:00", OP_EQ, tbuf); + } + + { + retval = parse_rfc1123_time("Mon, 20 Apr 2015 19:22:00 UTC", + ¤t_time); + tt_int_op(retval, ==, 0); + valid_until_time = get_state_valid_until_time(current_time); + + format_iso_time(tbuf, valid_until_time); + tt_str_op("2015-04-21 00:00:00", OP_EQ, tbuf); + } + + { + retval = parse_rfc1123_time("Mon, 20 Apr 2015 23:59:00 UTC", + ¤t_time); + tt_int_op(retval, ==, 0); + valid_until_time = get_state_valid_until_time(current_time); + + format_iso_time(tbuf, valid_until_time); + tt_str_op("2015-04-21 00:00:00", OP_EQ, tbuf); + } + + { + retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:00 UTC", + ¤t_time); + tt_int_op(retval, ==, 0); + valid_until_time = get_state_valid_until_time(current_time); + + format_iso_time(tbuf, valid_until_time); + tt_str_op("2015-04-21 00:00:00", OP_EQ, tbuf); + } + + done: + ; +} + +/* Mock function to immediately return our local 'mock_consensus'. */ +static networkstatus_t * +mock_networkstatus_get_live_consensus(time_t now) +{ + (void) now; + return mock_consensus; +} + +/** Test the get_next_valid_after_time() function. */ +static void +test_get_next_valid_after_time(void *arg) +{ + time_t current_time; + time_t valid_after_time; + char tbuf[ISO_TIME_LEN + 1]; + int retval; + + (void) arg; + + { + /* Setup a fake consensus just to get the times out of it, since + get_next_valid_after_time() needs them. */ + mock_consensus = tor_malloc_zero(sizeof(networkstatus_t)); + + retval = parse_rfc1123_time("Mon, 13 Jan 2016 16:00:00 UTC", + &mock_consensus->fresh_until); + tt_int_op(retval, ==, 0); + + retval = parse_rfc1123_time("Mon, 13 Jan 2016 15:00:00 UTC", + &mock_consensus->valid_after); + tt_int_op(retval, ==, 0); + + MOCK(networkstatus_get_live_consensus, + mock_networkstatus_get_live_consensus); + } + + { + /* Get the valid after time if called at 00:00:00 */ + retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:00 UTC", + ¤t_time); + tt_int_op(retval, ==, 0); + valid_after_time = get_next_valid_after_time(current_time); + + /* Compare it with the correct result */ + format_iso_time(tbuf, valid_after_time); + tt_str_op("2015-04-20 01:00:00", OP_EQ, tbuf); + } + + { + /* Get the valid until time if called at 00:00:01 */ + retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:01 UTC", + ¤t_time); + tt_int_op(retval, ==, 0); + valid_after_time = get_next_valid_after_time(current_time); + + /* Compare it with the correct result */ + format_iso_time(tbuf, valid_after_time); + tt_str_op("2015-04-20 01:00:00", OP_EQ, tbuf); + } + + { + retval = parse_rfc1123_time("Mon, 20 Apr 2015 23:30:01 UTC", + ¤t_time); + tt_int_op(retval, ==, 0); + valid_after_time = get_next_valid_after_time(current_time); + + /* Compare it with the correct result */ + format_iso_time(tbuf, valid_after_time); + tt_str_op("2015-04-21 00:00:00", OP_EQ, tbuf); + } + + done: + networkstatus_vote_free(mock_consensus); +} + +/* In this test we are going to generate a sr_commit_t object and validate + * it. We first generate our values, and then we parse them as if they were + * received from the network. After we parse both the commit and the reveal, + * we verify that they indeed match. */ +static void +test_sr_commit(void *arg) +{ + authority_cert_t *auth_cert = NULL; + time_t now = time(NULL); + sr_commit_t *our_commit = NULL; + smartlist_t *args = smartlist_new(); + + (void) arg; + + { /* Setup a minimal dirauth environment for this test */ + or_options_t *options = get_options_mutable(); + + auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + tt_assert(auth_cert); + + options->AuthoritativeDir = 1; + tt_int_op(0, ==, load_ed_keys(options, now)); + } + + /* Generate our commit object and validate it has the appropriate field + * that we can then use to build a representation that we'll find in a + * vote coming from the network. */ + { + sr_commit_t test_commit; + our_commit = sr_generate_our_commit(now, auth_cert); + tt_assert(our_commit); + /* Default and only supported algorithm for now. */ + tt_assert(our_commit->alg == DIGEST_SHA3_256); + /* We should have a reveal value. */ + tt_assert(commit_has_reveal_value(our_commit)); + /* We should have a random value. */ + tt_assert(!tor_mem_is_zero((char *) our_commit->random_number, + sizeof(our_commit->random_number))); + /* Commit and reveal timestamp should be the same. */ + tt_u64_op(our_commit->commit_ts, ==, our_commit->reveal_ts); + /* We should have a hashed reveal. */ + tt_assert(!tor_mem_is_zero(our_commit->hashed_reveal, + sizeof(our_commit->hashed_reveal))); + /* Do we have a valid encoded commit and reveal. Note the following only + * tests if the generated values are correct. Their could be a bug in + * the decode function but we test them seperately. */ + tt_int_op(0, ==, reveal_decode(our_commit->encoded_reveal, + &test_commit)); + tt_int_op(0, ==, commit_decode(our_commit->encoded_commit, + &test_commit)); + tt_int_op(0, ==, verify_commit_and_reveal(our_commit)); + } + + /* Let's make sure our verify commit and reveal function works. We'll + * make it fail a bit with known failure case. */ + { + /* Copy our commit so we don't alter it for the rest of testing. */ + sr_commit_t test_commit; + memcpy(&test_commit, our_commit, sizeof(test_commit)); + + /* Timestamp MUST match. */ + test_commit.commit_ts = test_commit.reveal_ts - 42; + tt_int_op(-1, ==, verify_commit_and_reveal(&test_commit)); + memcpy(&test_commit, our_commit, sizeof(test_commit)); + tt_int_op(0, ==, verify_commit_and_reveal(&test_commit)); + + /* Hashed reveal must match the H(encoded_reveal). */ + memset(test_commit.hashed_reveal, 'X', + sizeof(test_commit.hashed_reveal)); + tt_int_op(-1, ==, verify_commit_and_reveal(&test_commit)); + memcpy(&test_commit, our_commit, sizeof(test_commit)); + tt_int_op(0, ==, verify_commit_and_reveal(&test_commit)); + } + + /* We'll build a list of values from our commit that our parsing function + * takes from a vote line and see if we can parse it correctly. */ + { + sr_commit_t *parsed_commit; + smartlist_add(args, tor_strdup("1")); + smartlist_add(args, + tor_strdup(crypto_digest_algorithm_get_name(our_commit->alg))); + smartlist_add(args, tor_strdup(sr_commit_get_rsa_fpr(our_commit))); + smartlist_add(args, our_commit->encoded_commit); + smartlist_add(args, our_commit->encoded_reveal); + parsed_commit = sr_parse_commit(args); + tt_assert(parsed_commit); + /* That parsed commit should be _EXACTLY_ like our original commit (we + * have to explicitly set the valid flag though). */ + parsed_commit->valid = 1; + tt_mem_op(parsed_commit, OP_EQ, our_commit, sizeof(*parsed_commit)); + /* Cleanup */ + tor_free(smartlist_get(args, 0)); /* strdup here. */ + tor_free(smartlist_get(args, 1)); /* strdup here. */ + smartlist_clear(args); + sr_commit_free(parsed_commit); + } + + done: + smartlist_free(args); + sr_commit_free(our_commit); +} + +/* Test the encoding and decoding function for commit and reveal values. */ +static void +test_encoding(void *arg) +{ + (void) arg; + int ret, duper_rand = 42; + /* Random number is 32 bytes. */ + char raw_rand[32]; + time_t ts = 1454333590; + char hashed_rand[DIGEST256_LEN], hashed_reveal[DIGEST256_LEN]; + sr_commit_t parsed_commit; + + /* Encoded commit is: base64-encode( 1454333590 || H(H(42)) ). Remember + * that we do no expose the raw bytes of our PRNG to the network thus + * explaining the double H(). */ + static const char *encoded_commit = + "AAAAAFavXpZbx2LRneYFSLPCP8DLp9BXfeH5FXzbkxM4iRXKGeA54g=="; + /* Encoded reveal is: base64-encode( 1454333590 || H(42) ). */ + static const char *encoded_reveal = + "AAAAAFavXpYk9x9kTjiQWUqjHwSAEOdPAfCaurXgjPy173SzYjeC2g=="; + + /* Set up our raw random bytes array. */ + memset(raw_rand, 0, sizeof(raw_rand)); + memcpy(raw_rand, &duper_rand, sizeof(duper_rand)); + /* Hash random number. */ + ret = crypto_digest256(hashed_rand, raw_rand, + sizeof(raw_rand), SR_DIGEST_ALG); + tt_int_op(0, ==, ret); + /* Hash reveal value. */ + tt_int_op(SR_REVEAL_BASE64_LEN, ==, strlen(encoded_reveal)); + ret = crypto_digest256(hashed_reveal, encoded_reveal, + strlen(encoded_reveal), SR_DIGEST_ALG); + tt_int_op(0, ==, ret); + tt_int_op(SR_COMMIT_BASE64_LEN, ==, strlen(encoded_commit)); + + /* Test our commit/reveal decode functions. */ + { + /* Test the reveal encoded value. */ + tt_int_op(0, ==, reveal_decode(encoded_reveal, &parsed_commit)); + tt_u64_op(ts, ==, parsed_commit.reveal_ts); + tt_mem_op(hashed_rand, OP_EQ, parsed_commit.random_number, + sizeof(hashed_rand)); + + /* Test the commit encoded value. */ + memset(&parsed_commit, 0, sizeof(parsed_commit)); + tt_int_op(0, ==, commit_decode(encoded_commit, &parsed_commit)); + tt_u64_op(ts, ==, parsed_commit.commit_ts); + tt_mem_op(encoded_commit, OP_EQ, parsed_commit.encoded_commit, + sizeof(parsed_commit.encoded_commit)); + tt_mem_op(hashed_reveal, OP_EQ, parsed_commit.hashed_reveal, + sizeof(hashed_reveal)); + } + + /* Test our commit/reveal encode functions. */ + { + /* Test the reveal encode. */ + char encoded[SR_REVEAL_BASE64_LEN + 1]; + parsed_commit.reveal_ts = ts; + memcpy(parsed_commit.random_number, hashed_rand, + sizeof(parsed_commit.random_number)); + ret = reveal_encode(&parsed_commit, encoded, sizeof(encoded)); + tt_int_op(SR_REVEAL_BASE64_LEN, ==, ret); + tt_mem_op(encoded_reveal, OP_EQ, encoded, strlen(encoded_reveal)); + } + + { + /* Test the commit encode. */ + char encoded[SR_COMMIT_BASE64_LEN + 1]; + parsed_commit.commit_ts = ts; + memcpy(parsed_commit.hashed_reveal, hashed_reveal, + sizeof(parsed_commit.hashed_reveal)); + ret = commit_encode(&parsed_commit, encoded, sizeof(encoded)); + tt_int_op(SR_COMMIT_BASE64_LEN, ==, ret); + tt_mem_op(encoded_commit, OP_EQ, encoded, strlen(encoded_commit)); + } + + done: + ; +} + +/** Setup some SRVs in our SR state. If <b>also_current</b> is set, then set + * both current and previous SRVs. + * Helper of test_vote() and test_sr_compute_srv(). */ +static void +test_sr_setup_srv(int also_current) +{ + sr_srv_t *srv = tor_malloc_zero(sizeof(sr_srv_t)); + srv->num_reveals = 42; + memcpy(srv->value, + "ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ", + sizeof(srv->value)); + + sr_state_set_previous_srv(srv); + + if (also_current) { + srv = tor_malloc_zero(sizeof(sr_srv_t)); + srv->num_reveals = 128; + memcpy(srv->value, + "NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN", + sizeof(srv->value)); + + sr_state_set_current_srv(srv); + } +} + +/* Test anything that has to do with SR protocol and vote. */ +static void +test_vote(void *arg) +{ + int ret; + time_t now = time(NULL); + sr_commit_t *our_commit = NULL; + + (void) arg; + + MOCK(trusteddirserver_get_by_v3_auth_digest, + trusteddirserver_get_by_v3_auth_digest_m); + + { /* Setup a minimal dirauth environment for this test */ + init_authority_state(); + /* Set ourself in reveal phase so we can parse the reveal value in the + * vote as well. */ + set_sr_phase(SR_PHASE_REVEAL); + } + + /* Generate our commit object and validate it has the appropriate field + * that we can then use to build a representation that we'll find in a + * vote coming from the network. */ + { + sr_commit_t *saved_commit; + our_commit = sr_generate_our_commit(now, mock_cert); + tt_assert(our_commit); + sr_state_add_commit(our_commit); + /* Make sure it's there. */ + saved_commit = sr_state_get_commit(our_commit->rsa_identity); + tt_assert(saved_commit); + } + + /* Also setup the SRVs */ + test_sr_setup_srv(1); + + { /* Now test the vote generation */ + smartlist_t *chunks = smartlist_new(); + smartlist_t *tokens = smartlist_new(); + /* Get our vote line and validate it. */ + char *lines = sr_get_string_for_vote(); + tt_assert(lines); + /* Split the lines. We expect 2 here. */ + ret = smartlist_split_string(chunks, lines, "\n", SPLIT_IGNORE_BLANK, 0); + tt_int_op(ret, ==, 4); + tt_str_op(smartlist_get(chunks, 0), OP_EQ, "shared-rand-participate"); + /* Get our commitment line and will validate it agains our commit. The + * format is as follow: + * "shared-rand-commitment" SP version SP algname SP identity + * SP COMMIT [SP REVEAL] NL + */ + char *commit_line = smartlist_get(chunks, 1); + tt_assert(commit_line); + ret = smartlist_split_string(tokens, commit_line, " ", 0, 0); + tt_int_op(ret, ==, 6); + tt_str_op(smartlist_get(tokens, 0), OP_EQ, "shared-rand-commit"); + tt_str_op(smartlist_get(tokens, 1), OP_EQ, "1"); + tt_str_op(smartlist_get(tokens, 2), OP_EQ, + crypto_digest_algorithm_get_name(DIGEST_SHA3_256)); + char digest[DIGEST_LEN]; + base16_decode(digest, sizeof(digest), smartlist_get(tokens, 3), + HEX_DIGEST_LEN); + tt_mem_op(digest, ==, our_commit->rsa_identity, sizeof(digest)); + tt_str_op(smartlist_get(tokens, 4), OP_EQ, our_commit->encoded_commit); + tt_str_op(smartlist_get(tokens, 5), OP_EQ, our_commit->encoded_reveal); + + /* Finally, does this vote line creates a valid commit object? */ + smartlist_t *args = smartlist_new(); + smartlist_add(args, smartlist_get(tokens, 1)); + smartlist_add(args, smartlist_get(tokens, 2)); + smartlist_add(args, smartlist_get(tokens, 3)); + smartlist_add(args, smartlist_get(tokens, 4)); + smartlist_add(args, smartlist_get(tokens, 5)); + sr_commit_t *parsed_commit = sr_parse_commit(args); + tt_assert(parsed_commit); + /* Set valid flag explicitly here to compare since it's not set by + * simply parsing the commit. */ + parsed_commit->valid = 1; + tt_mem_op(parsed_commit, ==, our_commit, sizeof(*our_commit)); + + /* minor cleanup */ + SMARTLIST_FOREACH(tokens, char *, s, tor_free(s)); + smartlist_clear(tokens); + + /* Now test the previous SRV */ + char *prev_srv_line = smartlist_get(chunks, 2); + tt_assert(prev_srv_line); + ret = smartlist_split_string(tokens, prev_srv_line, " ", 0, 0); + tt_int_op(ret, ==, 3); + tt_str_op(smartlist_get(tokens, 0), OP_EQ, "shared-rand-previous-value"); + tt_str_op(smartlist_get(tokens, 1), OP_EQ, "42"); + tt_str_op(smartlist_get(tokens, 2), OP_EQ, + "WlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlo="); + + /* minor cleanup */ + SMARTLIST_FOREACH(tokens, char *, s, tor_free(s)); + smartlist_clear(tokens); + + /* Now test the current SRV */ + char *current_srv_line = smartlist_get(chunks, 3); + tt_assert(current_srv_line); + ret = smartlist_split_string(tokens, current_srv_line, " ", 0, 0); + tt_int_op(ret, ==, 3); + tt_str_op(smartlist_get(tokens, 0), OP_EQ, "shared-rand-current-value"); + tt_str_op(smartlist_get(tokens, 1), OP_EQ, "128"); + tt_str_op(smartlist_get(tokens, 2), OP_EQ, + "Tk5OTk5OTk5OTk5OTk5OTk5OTk5OTk5OTk5OTk5OTk4="); + + /* Clean up */ + sr_commit_free(parsed_commit); + SMARTLIST_FOREACH(chunks, char *, s, tor_free(s)); + smartlist_free(chunks); + SMARTLIST_FOREACH(tokens, char *, s, tor_free(s)); + smartlist_free(tokens); + smartlist_clear(args); + smartlist_free(args); + } + + done: + sr_commit_free(our_commit); + UNMOCK(trusteddirserver_get_by_v3_auth_digest); +} + +static const char *sr_state_str = "Version 1\n" + "TorVersion 0.2.9.0-alpha-dev\n" + "ValidAfter 2037-04-19 07:16:00\n" + "ValidUntil 2037-04-20 07:16:00\n" + "Commit 1 sha3-256 FA3CEC2C99DC68D3166B9B6E4FA21A4026C2AB1C " + "7M8GdubCAAdh7WUG0DiwRyxTYRKji7HATa7LLJEZ/UAAAAAAVmfUSg== " + "AAAAAFZn1EojfIheIw42bjK3VqkpYyjsQFSbv/dxNna3Q8hUEPKpOw==\n" + "Commit 1 sha3-256 41E89EDFBFBA44983E21F18F2230A4ECB5BFB543 " + "17aUsYuMeRjd2N1r8yNyg7aHqRa6gf4z7QPoxxAZbp0AAAAAVmfUSg==\n" + "Commit 1 sha3-256 36637026573A04110CF3E6B1D201FB9A98B88734 " + "DDDYtripvdOU+XPEUm5xpU64d9IURSds1xSwQsgeB8oAAAAAVmfUSg==\n" + "SharedRandPreviousValue 4 qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqo=\n" + "SharedRandCurrentValue 3 8dWeW12KEzTGEiLGgO1UVJ7Z91CekoRcxt6Q9KhnOFI=\n"; + +/** Create an SR disk state, parse it and validate that the parsing went + * well. Yes! */ +static void +test_state_load_from_disk(void *arg) +{ + int ret; + char *dir = tor_strdup(get_fname("test_sr_state")); + char *sr_state_path = tor_strdup(get_fname("test_sr_state/sr_state")); + sr_state_t *the_sr_state = NULL; + + (void) arg; + + MOCK(trusteddirserver_get_by_v3_auth_digest, + trusteddirserver_get_by_v3_auth_digest_m); + + /* First try with a nonexistent path. */ + ret = disk_state_load_from_disk_impl("NONEXISTENTNONEXISTENT"); + tt_assert(ret == -ENOENT); + + /* Now create a mock state directory and state file */ +#ifdef _WIN32 + ret = mkdir(dir); +#else + ret = mkdir(dir, 0700); +#endif + tt_assert(ret == 0); + ret = write_str_to_file(sr_state_path, sr_state_str, 0); + tt_assert(ret == 0); + + /* Try to load the directory itself. Should fail. */ + ret = disk_state_load_from_disk_impl(dir); +#ifdef _WIN32 + tt_int_op(ret, OP_EQ, -EACCES); +#else + tt_int_op(ret, OP_EQ, -EISDIR); +#endif + + /* State should be non-existent at this point. */ + the_sr_state = get_sr_state(); + tt_assert(!the_sr_state); + + /* Now try to load the correct file! */ + ret = disk_state_load_from_disk_impl(sr_state_path); + tt_assert(ret == 0); + + /* Check the content of the state */ + /* XXX check more deeply!!! */ + the_sr_state = get_sr_state(); + tt_assert(the_sr_state); + tt_assert(the_sr_state->version == 1); + tt_assert(digestmap_size(the_sr_state->commits) == 3); + tt_assert(the_sr_state->current_srv); + tt_assert(the_sr_state->current_srv->num_reveals == 3); + tt_assert(the_sr_state->previous_srv); + + /* XXX Now also try loading corrupted state files and make sure parsing + fails */ + + done: + tor_free(dir); + tor_free(sr_state_path); + UNMOCK(trusteddirserver_get_by_v3_auth_digest); +} + +/** Generate three specially crafted commits (based on the test + * vector at sr_srv_calc_ref.py). Helper of test_sr_compute_srv(). */ +static void +test_sr_setup_commits(void) +{ + time_t now = time(NULL); + sr_commit_t *commit_a, *commit_b, *commit_c, *commit_d; + sr_commit_t *place_holder = tor_malloc_zero(sizeof(*place_holder)); + authority_cert_t *auth_cert = NULL; + + { /* Setup a minimal dirauth environment for this test */ + or_options_t *options = get_options_mutable(); + + auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + tt_assert(auth_cert); + + options->AuthoritativeDir = 1; + tt_int_op(0, ==, load_ed_keys(options, now)); + } + + /* Generate three dummy commits according to sr_srv_calc_ref.py . Then + register them to the SR state. Also register a fourth commit 'd' with no + reveal info, to make sure that it will get ignored during SRV + calculation. */ + + { /* Commit from auth 'a' */ + commit_a = sr_generate_our_commit(now, auth_cert); + tt_assert(commit_a); + + /* Do some surgery on the commit */ + memset(commit_a->rsa_identity, 'A', sizeof(commit_a->rsa_identity)); + base16_encode(commit_a->rsa_identity_hex, + sizeof(commit_a->rsa_identity_hex), commit_a->rsa_identity, + sizeof(commit_a->rsa_identity)); + strlcpy(commit_a->encoded_reveal, + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + sizeof(commit_a->encoded_reveal)); + memcpy(commit_a->hashed_reveal, + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + sizeof(commit_a->hashed_reveal)); + } + + { /* Commit from auth 'b' */ + commit_b = sr_generate_our_commit(now, auth_cert); + tt_assert(commit_b); + + /* Do some surgery on the commit */ + memset(commit_b->rsa_identity, 'B', sizeof(commit_b->rsa_identity)); + base16_encode(commit_b->rsa_identity_hex, + sizeof(commit_b->rsa_identity_hex), commit_b->rsa_identity, + sizeof(commit_b->rsa_identity)); + strlcpy(commit_b->encoded_reveal, + "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB", + sizeof(commit_b->encoded_reveal)); + memcpy(commit_b->hashed_reveal, + "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB", + sizeof(commit_b->hashed_reveal)); + } + + { /* Commit from auth 'c' */ + commit_c = sr_generate_our_commit(now, auth_cert); + tt_assert(commit_c); + + /* Do some surgery on the commit */ + memset(commit_c->rsa_identity, 'C', sizeof(commit_c->rsa_identity)); + base16_encode(commit_c->rsa_identity_hex, + sizeof(commit_c->rsa_identity_hex), commit_c->rsa_identity, + sizeof(commit_c->rsa_identity)); + strlcpy(commit_c->encoded_reveal, + "CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC", + sizeof(commit_c->encoded_reveal)); + memcpy(commit_c->hashed_reveal, + "CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC", + sizeof(commit_c->hashed_reveal)); + } + + { /* Commit from auth 'd' */ + commit_d = sr_generate_our_commit(now, auth_cert); + tt_assert(commit_d); + + /* Do some surgery on the commit */ + memset(commit_d->rsa_identity, 'D', sizeof(commit_d->rsa_identity)); + base16_encode(commit_d->rsa_identity_hex, + sizeof(commit_d->rsa_identity_hex), commit_d->rsa_identity, + sizeof(commit_d->rsa_identity)); + strlcpy(commit_d->encoded_reveal, + "DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD", + sizeof(commit_d->encoded_reveal)); + memcpy(commit_d->hashed_reveal, + "DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD", + sizeof(commit_d->hashed_reveal)); + /* Clean up its reveal info */ + memcpy(place_holder, commit_d, sizeof(*place_holder)); + memset(commit_d->encoded_reveal, 0, sizeof(commit_d->encoded_reveal)); + tt_assert(!commit_has_reveal_value(commit_d)); + } + + /* Register commits to state (during commit phase) */ + set_sr_phase(SR_PHASE_COMMIT); + save_commit_to_state(commit_a); + save_commit_to_state(commit_b); + save_commit_to_state(commit_c); + save_commit_to_state(commit_d); + tt_int_op(digestmap_size(get_sr_state()->commits), ==, 4); + + /* Now during REVEAL phase save commit D by restoring its reveal. */ + set_sr_phase(SR_PHASE_REVEAL); + save_commit_to_state(place_holder); + tt_str_op(commit_d->encoded_reveal, OP_EQ, + "DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD"); + /* Go back to an empty encoded reveal value. */ + memset(commit_d->encoded_reveal, 0, sizeof(commit_d->encoded_reveal)); + memset(commit_d->random_number, 0, sizeof(commit_d->random_number)); + tt_assert(!commit_has_reveal_value(commit_d)); + + done: + return; +} + +/** Verify that the SRV generation procedure is proper by testing it against + * the test vector from ./sr_srv_calc_ref.py. */ +static void +test_sr_compute_srv(void *arg) +{ + (void) arg; + const sr_srv_t *current_srv = NULL; + +#define SRV_TEST_VECTOR \ + "2A9B1D6237DAB312A40F575DA85C147663E7ED3F80E9555395F15B515C74253D" + + MOCK(trusteddirserver_get_by_v3_auth_digest, + trusteddirserver_get_by_v3_auth_digest_m); + + init_authority_state(); + + /* Setup the commits for this unittest */ + test_sr_setup_commits(); + test_sr_setup_srv(0); + + /* Now switch to reveal phase */ + set_sr_phase(SR_PHASE_REVEAL); + + /* Compute the SRV */ + sr_compute_srv(); + + /* Check the result against the test vector */ + current_srv = sr_state_get_current_srv(); + tt_assert(current_srv); + tt_u64_op(current_srv->num_reveals, ==, 3); + tt_str_op(hex_str((char*)current_srv->value, 32), + ==, + SRV_TEST_VECTOR); + + done: + UNMOCK(trusteddirserver_get_by_v3_auth_digest); +} + +/** Return a minimal vote document with a current SRV value set to + * <b>srv</b>. */ +static networkstatus_t * +get_test_vote_with_curr_srv(const char *srv) +{ + networkstatus_t *vote = tor_malloc_zero(sizeof(networkstatus_t)); + + vote->type = NS_TYPE_VOTE; + vote->sr_info.participate = 1; + vote->sr_info.current_srv = tor_malloc_zero(sizeof(sr_srv_t)); + vote->sr_info.current_srv->num_reveals = 42; + memcpy(vote->sr_info.current_srv->value, + srv, + sizeof(vote->sr_info.current_srv->value)); + + return vote; +} + +/* Test the function that picks the right SRV given a bunch of votes. Make sure + * that the function returns an SRV iff the majority/agreement requirements are + * met. */ +static void +test_sr_get_majority_srv_from_votes(void *arg) +{ + sr_srv_t *chosen_srv; + smartlist_t *votes = smartlist_new(); + +#define SRV_1 "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" +#define SRV_2 "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB" + + (void) arg; + + init_authority_state(); + /* Make sure our SRV is fresh so we can consider the super majority with + * the consensus params of number of agreements needed. */ + sr_state_set_fresh_srv(); + + /* The test relies on the dirauth list being initialized. */ + clear_dir_servers(); + add_default_trusted_dir_authorities(V3_DIRINFO); + tt_int_op(get_n_authorities(V3_DIRINFO), ==, 9); + + { /* Prepare voting environment with just a single vote. */ + networkstatus_t *vote = get_test_vote_with_curr_srv(SRV_1); + smartlist_add(votes, vote); + } + + /* Since it's only one vote with an SRV, it should not achieve majority and + hence no SRV will be returned. */ + chosen_srv = get_majority_srv_from_votes(votes, 1); + tt_assert(!chosen_srv); + + { /* Now put in 8 more votes. Let SRV_1 have majority. */ + int i; + /* Now 7 votes believe in SRV_1 */ + for (i = 0; i < 3; i++) { + networkstatus_t *vote = get_test_vote_with_curr_srv(SRV_1); + smartlist_add(votes, vote); + } + /* and 2 votes believe in SRV_2 */ + for (i = 0; i < 2; i++) { + networkstatus_t *vote = get_test_vote_with_curr_srv(SRV_2); + smartlist_add(votes, vote); + } + for (i = 0; i < 3; i++) { + networkstatus_t *vote = get_test_vote_with_curr_srv(SRV_1); + smartlist_add(votes, vote); + } + + tt_int_op(smartlist_len(votes), ==, 9); + } + + /* Now we achieve majority for SRV_1, but not the AuthDirNumSRVAgreements + requirement. So still not picking an SRV. */ + set_num_srv_agreements(8); + chosen_srv = get_majority_srv_from_votes(votes, 1); + tt_assert(!chosen_srv); + + /* We will now lower the AuthDirNumSRVAgreements requirement by tweaking the + * consensus parameter and we will try again. This time it should work. */ + set_num_srv_agreements(7); + chosen_srv = get_majority_srv_from_votes(votes, 1); + tt_assert(chosen_srv); + tt_u64_op(chosen_srv->num_reveals, ==, 42); + tt_mem_op(chosen_srv->value, OP_EQ, SRV_1, sizeof(chosen_srv->value)); + + done: + SMARTLIST_FOREACH(votes, networkstatus_t *, vote, + networkstatus_vote_free(vote)); + smartlist_free(votes); +} + +static void +test_utils(void *arg) +{ + (void) arg; + + /* Testing srv_dup(). */ + { + sr_srv_t *srv = NULL, *dup_srv = NULL; + const char *srv_value = + "1BDB7C3E973936E4D13A49F37C859B3DC69C429334CF9412E3FEF6399C52D47A"; + srv = tor_malloc_zero(sizeof(*srv)); + srv->num_reveals = 42; + memcpy(srv->value, srv_value, sizeof(srv->value)); + dup_srv = srv_dup(srv); + tt_assert(dup_srv); + tt_u64_op(dup_srv->num_reveals, ==, srv->num_reveals); + tt_mem_op(dup_srv->value, OP_EQ, srv->value, sizeof(srv->value)); + tor_free(srv); + tor_free(dup_srv); + } + + /* Testing commitments_are_the_same(). Currently, the check is to test the + * value of the encoded commit so let's make sure that actually works. */ + { + /* Payload of 57 bytes that is the length of sr_commit_t->encoded_commit. + * 56 bytes of payload and a NUL terminated byte at the end ('\x00') + * which comes down to SR_COMMIT_BASE64_LEN + 1. */ + const char *payload = + "\x5d\xb9\x60\xb6\xcc\x51\x68\x52\x31\xd9\x88\x88\x71\x71\xe0\x30" + "\x59\x55\x7f\xcd\x61\xc0\x4b\x05\xb8\xcd\xc1\x48\xe9\xcd\x16\x1f" + "\x70\x15\x0c\xfc\xd3\x1a\x75\xd0\x93\x6c\xc4\xe0\x5c\xbe\xe2\x18" + "\xc7\xaf\x72\xb6\x7c\x9b\x52\x00"; + sr_commit_t commit1, commit2; + memcpy(commit1.encoded_commit, payload, sizeof(commit1.encoded_commit)); + memcpy(commit2.encoded_commit, payload, sizeof(commit2.encoded_commit)); + tt_int_op(commitments_are_the_same(&commit1, &commit2), ==, 1); + /* Let's corrupt one of them. */ + memset(commit1.encoded_commit, 'A', sizeof(commit1.encoded_commit)); + tt_int_op(commitments_are_the_same(&commit1, &commit2), ==, 0); + } + + /* Testing commit_is_authoritative(). */ + { + crypto_pk_t *k = crypto_pk_new(); + char digest[DIGEST_LEN]; + sr_commit_t commit; + + tt_assert(!crypto_pk_generate_key(k)); + + tt_int_op(0, ==, crypto_pk_get_digest(k, digest)); + memcpy(commit.rsa_identity, digest, sizeof(commit.rsa_identity)); + tt_int_op(commit_is_authoritative(&commit, digest), ==, 1); + /* Change the pubkey. */ + memset(commit.rsa_identity, 0, sizeof(commit.rsa_identity)); + tt_int_op(commit_is_authoritative(&commit, digest), ==, 0); + } + + /* Testing get_phase_str(). */ + { + tt_str_op(get_phase_str(SR_PHASE_REVEAL), ==, "reveal"); + tt_str_op(get_phase_str(SR_PHASE_COMMIT), ==, "commit"); + } + + /* Testing phase transition */ + { + init_authority_state(); + set_sr_phase(SR_PHASE_COMMIT); + tt_int_op(is_phase_transition(SR_PHASE_REVEAL), ==, 1); + tt_int_op(is_phase_transition(SR_PHASE_COMMIT), ==, 0); + set_sr_phase(SR_PHASE_REVEAL); + tt_int_op(is_phase_transition(SR_PHASE_REVEAL), ==, 0); + tt_int_op(is_phase_transition(SR_PHASE_COMMIT), ==, 1); + /* Junk. */ + tt_int_op(is_phase_transition(42), ==, 1); + } + + done: + return; +} + +static void +test_state_transition(void *arg) +{ + sr_state_t *state = NULL; + time_t now = time(NULL); + + (void) arg; + + { /* Setup a minimal dirauth environment for this test */ + init_authority_state(); + state = get_sr_state(); + tt_assert(state); + } + + /* Test our state reset for a new protocol run. */ + { + /* Add a commit to the state so we can test if the reset cleans the + * commits. Also, change all params that we expect to be updated. */ + sr_commit_t *commit = sr_generate_our_commit(now, mock_cert); + tt_assert(commit); + sr_state_add_commit(commit); + tt_int_op(digestmap_size(state->commits), ==, 1); + /* Let's test our delete feature. */ + sr_state_delete_commits(); + tt_int_op(digestmap_size(state->commits), ==, 0); + /* Add it back so we can continue the rest of the test because after + * deletiong our commit will be freed so generate a new one. */ + commit = sr_generate_our_commit(now, mock_cert); + tt_assert(commit); + sr_state_add_commit(commit); + tt_int_op(digestmap_size(state->commits), ==, 1); + state->n_reveal_rounds = 42; + state->n_commit_rounds = 43; + state->n_protocol_runs = 44; + reset_state_for_new_protocol_run(now); + tt_int_op(state->n_reveal_rounds, ==, 0); + tt_int_op(state->n_commit_rounds, ==, 0); + tt_u64_op(state->n_protocol_runs, ==, 45); + tt_int_op(digestmap_size(state->commits), ==, 0); + } + + /* Test SRV rotation in our state. */ + { + const sr_srv_t *cur, *prev; + test_sr_setup_srv(1); + cur = sr_state_get_current_srv(); + tt_assert(cur); + /* After, current srv should be the previous and then set to NULL. */ + state_rotate_srv(); + prev = sr_state_get_previous_srv(); + tt_assert(prev == cur); + tt_assert(!sr_state_get_current_srv()); + } + + /* New protocol run. */ + { + const sr_srv_t *cur; + /* Setup some new SRVs so we can confirm that a new protocol run + * actually makes them rotate and compute new ones. */ + test_sr_setup_srv(1); + cur = sr_state_get_current_srv(); + tt_assert(cur); + set_sr_phase(SR_PHASE_REVEAL); + MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); + new_protocol_run(now); + UNMOCK(get_my_v3_authority_cert); + /* Rotation happened. */ + tt_assert(sr_state_get_previous_srv() == cur); + /* We are going into COMMIT phase so we had to rotate our SRVs. Usually + * our current SRV would be NULL but a new protocol run should make us + * compute a new SRV. */ + tt_assert(sr_state_get_current_srv()); + /* Also, make sure we did change the current. */ + tt_assert(sr_state_get_current_srv() != cur); + /* We should have our commitment alone. */ + tt_int_op(digestmap_size(state->commits), ==, 1); + tt_int_op(state->n_reveal_rounds, ==, 0); + tt_int_op(state->n_commit_rounds, ==, 0); + /* 46 here since we were at 45 just before. */ + tt_u64_op(state->n_protocol_runs, ==, 46); + } + + /* Cleanup of SRVs. */ + { + sr_state_clean_srvs(); + tt_assert(!sr_state_get_current_srv()); + tt_assert(!sr_state_get_previous_srv()); + } + + done: + return; +} + +static void +test_keep_commit(void *arg) +{ + char fp[FINGERPRINT_LEN + 1]; + sr_commit_t *commit = NULL, *dup_commit = NULL; + sr_state_t *state; + time_t now = time(NULL); + + (void) arg; + + MOCK(trusteddirserver_get_by_v3_auth_digest, + trusteddirserver_get_by_v3_auth_digest_m); + + { /* Setup a minimal dirauth environment for this test */ + crypto_pk_t *k = crypto_pk_new(); + /* Have a key that is not the one from our commit. */ + tt_int_op(0, ==, crypto_pk_generate_key(k)); + tt_int_op(0, ==, crypto_pk_get_fingerprint(k, fp, 0)); + init_authority_state(); + state = get_sr_state(); + } + + /* Test this very important function that tells us if we should keep a + * commit or not in our state. Most of it depends on the phase and what's + * in the commit so we'll change the commit as we go. */ + commit = sr_generate_our_commit(now, mock_cert); + tt_assert(commit); + /* Set us in COMMIT phase for starter. */ + set_sr_phase(SR_PHASE_COMMIT); + /* We should never keep a commit from a non authoritative authority. */ + tt_int_op(should_keep_commit(commit, fp, SR_PHASE_COMMIT), ==, 0); + /* This should NOT be kept because it has a reveal value in it. */ + tt_assert(commit_has_reveal_value(commit)); + tt_int_op(should_keep_commit(commit, commit->rsa_identity, + SR_PHASE_COMMIT), ==, 0); + /* Add it to the state which should return to not keep it. */ + sr_state_add_commit(commit); + tt_int_op(should_keep_commit(commit, commit->rsa_identity, + SR_PHASE_COMMIT), ==, 0); + /* Remove it from state so we can continue our testing. */ + digestmap_remove(state->commits, commit->rsa_identity); + /* Let's remove our reveal value which should make it OK to keep it. */ + memset(commit->encoded_reveal, 0, sizeof(commit->encoded_reveal)); + tt_int_op(should_keep_commit(commit, commit->rsa_identity, + SR_PHASE_COMMIT), ==, 1); + + /* Let's reset our commit and go into REVEAL phase. */ + sr_commit_free(commit); + commit = sr_generate_our_commit(now, mock_cert); + tt_assert(commit); + /* Dup the commit so we have one with and one without a reveal value. */ + dup_commit = tor_malloc_zero(sizeof(*dup_commit)); + memcpy(dup_commit, commit, sizeof(*dup_commit)); + memset(dup_commit->encoded_reveal, 0, sizeof(dup_commit->encoded_reveal)); + set_sr_phase(SR_PHASE_REVEAL); + /* We should never keep a commit from a non authoritative authority. */ + tt_int_op(should_keep_commit(commit, fp, SR_PHASE_REVEAL), ==, 0); + /* We shouldn't accept a commit that is not in our state. */ + tt_int_op(should_keep_commit(commit, commit->rsa_identity, + SR_PHASE_REVEAL), ==, 0); + /* Important to add the commit _without_ the reveal here. */ + sr_state_add_commit(dup_commit); + tt_int_op(digestmap_size(state->commits), ==, 1); + /* Our commit should be valid that is authoritative, contains a reveal, be + * in the state and commitment and reveal values match. */ + tt_int_op(should_keep_commit(commit, commit->rsa_identity, + SR_PHASE_REVEAL), ==, 1); + /* The commit shouldn't be kept if it's not verified that is no matchin + * hashed reveal. */ + { + /* Let's save the hash reveal so we can restore it. */ + sr_commit_t place_holder; + memcpy(place_holder.hashed_reveal, commit->hashed_reveal, + sizeof(place_holder.hashed_reveal)); + memset(commit->hashed_reveal, 0, sizeof(commit->hashed_reveal)); + tt_int_op(should_keep_commit(commit, commit->rsa_identity, + SR_PHASE_REVEAL), ==, 0); + memcpy(commit->hashed_reveal, place_holder.hashed_reveal, + sizeof(commit->hashed_reveal)); + } + /* We shouldn't keep a commit that has no reveal. */ + tt_int_op(should_keep_commit(dup_commit, dup_commit->rsa_identity, + SR_PHASE_REVEAL), ==, 0); + /* We must not keep a commit that is not the same from the commit phase. */ + memset(commit->encoded_commit, 0, sizeof(commit->encoded_commit)); + tt_int_op(should_keep_commit(commit, commit->rsa_identity, + SR_PHASE_REVEAL), ==, 0); + + done: + sr_commit_free(commit); + sr_commit_free(dup_commit); + UNMOCK(trusteddirserver_get_by_v3_auth_digest); +} + +static void +test_state_update(void *arg) +{ + time_t commit_phase_time = 1452076000; + time_t reveal_phase_time = 1452086800; + sr_state_t *state; + + (void) arg; + + { + init_authority_state(); + state = get_sr_state(); + set_sr_phase(SR_PHASE_COMMIT); + /* We'll cheat a bit here and reset the creation time of the state which + * will avoid us to compute a valid_after time that fits the commit + * phase. */ + state->valid_after = 0; + state->n_reveal_rounds = 0; + state->n_commit_rounds = 0; + state->n_protocol_runs = 0; + } + + /* We need to mock for the state update function call. */ + MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); + + /* We are in COMMIT phase here and we'll trigger a state update but no + * transition. */ + sr_state_update(commit_phase_time); + tt_int_op(state->valid_after, ==, commit_phase_time); + tt_int_op(state->n_commit_rounds, ==, 1); + tt_int_op(state->phase, ==, SR_PHASE_COMMIT); + tt_int_op(digestmap_size(state->commits), ==, 1); + + /* We are still in the COMMIT phase here but we'll trigger a state + * transition to the REVEAL phase. */ + sr_state_update(reveal_phase_time); + tt_int_op(state->phase, ==, SR_PHASE_REVEAL); + tt_int_op(state->valid_after, ==, reveal_phase_time); + /* Only our commit should be in there. */ + tt_int_op(digestmap_size(state->commits), ==, 1); + tt_int_op(state->n_reveal_rounds, ==, 1); + + /* We can't update a state with a valid after _lower_ than the creation + * time so here it is. */ + sr_state_update(commit_phase_time); + tt_int_op(state->valid_after, ==, reveal_phase_time); + + /* Finally, let's go back in COMMIT phase so we can test the state update + * of a new protocol run. */ + state->valid_after = 0; + sr_state_update(commit_phase_time); + tt_int_op(state->valid_after, ==, commit_phase_time); + tt_int_op(state->n_commit_rounds, ==, 1); + tt_int_op(state->n_reveal_rounds, ==, 0); + tt_u64_op(state->n_protocol_runs, ==, 1); + tt_int_op(state->phase, ==, SR_PHASE_COMMIT); + tt_int_op(digestmap_size(state->commits), ==, 1); + tt_assert(state->current_srv); + + done: + sr_state_free(); + UNMOCK(get_my_v3_authority_cert); +} + +struct testcase_t sr_tests[] = { + { "get_sr_protocol_phase", test_get_sr_protocol_phase, TT_FORK, + NULL, NULL }, + { "sr_commit", test_sr_commit, TT_FORK, + NULL, NULL }, + { "keep_commit", test_keep_commit, TT_FORK, + NULL, NULL }, + { "encoding", test_encoding, TT_FORK, + NULL, NULL }, + { "get_next_valid_after_time", test_get_next_valid_after_time, TT_FORK, + NULL, NULL }, + { "get_state_valid_until_time", test_get_state_valid_until_time, TT_FORK, + NULL, NULL }, + { "vote", test_vote, TT_FORK, + NULL, NULL }, + { "state_load_from_disk", test_state_load_from_disk, TT_FORK, + NULL, NULL }, + { "sr_compute_srv", test_sr_compute_srv, TT_FORK, NULL, NULL }, + { "sr_get_majority_srv_from_votes", test_sr_get_majority_srv_from_votes, + TT_FORK, NULL, NULL }, + { "utils", test_utils, TT_FORK, NULL, NULL }, + { "state_transition", test_state_transition, TT_FORK, NULL, NULL }, + { "state_update", test_state_update, TT_FORK, + NULL, NULL }, + END_OF_TESTCASES +}; + diff --git a/src/test/test_slow.c b/src/test/test_slow.c index c1d2e81914..7c9f0b1cc2 100644 --- a/src/test/test_slow.c +++ b/src/test/test_slow.c @@ -18,9 +18,6 @@ #include "or.h" #include "test.h" -extern struct testcase_t slow_crypto_tests[]; -extern struct testcase_t slow_util_tests[]; - struct testgroup_t testgroups[] = { { "slow/crypto/", slow_crypto_tests }, { "slow/util/", slow_util_tests }, diff --git a/src/test/test_socks.c b/src/test/test_socks.c index 6da09fd653..62ff12fe15 100644 --- a/src/test/test_socks.c +++ b/src/test/test_socks.c @@ -34,7 +34,7 @@ socks_test_cleanup(const struct testcase_t *testcase, void *ptr) return 1; } -const struct testcase_setup_t socks_setup = { +static const struct testcase_setup_t socks_setup = { socks_test_setup, socks_test_cleanup }; diff --git a/src/test/test_status.c b/src/test/test_status.c index 84a0f6c024..b4438aabe9 100644 --- a/src/test/test_status.c +++ b/src/test/test_status.c @@ -310,8 +310,6 @@ NS_DECL(void, logv, (int severity, log_domain_mask_t domain, NS_DECL(int, server_mode, (const or_options_t *options)); static routerinfo_t *mock_routerinfo; -extern int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1]; -extern int onion_handshakes_assigned[MAX_ONION_HANDSHAKE_TYPE+1]; static void NS(test_main)(void *arg) diff --git a/src/test/test_switch_id.sh b/src/test/test_switch_id.sh index 1b4e0998b5..79c44f2eb1 100755 --- a/src/test/test_switch_id.sh +++ b/src/test/test_switch_id.sh @@ -10,6 +10,10 @@ if test "`id -u nobody`" = ""; then exit 1 fi +if test "$OVERRIDE_GCDA_PERMISSIONS_HACK" = "yes"; then + find src -type f -name '*gcda' -print0 | xargs -0 chmod 0666 +fi + "${builddir:-.}/src/test/test-switch-id" nobody setuid || exit 1 "${builddir:-.}/src/test/test-switch-id" nobody root-bind-low || exit 1 "${builddir:-.}/src/test/test-switch-id" nobody setuid-strict || exit 1 @@ -19,6 +23,9 @@ fi "${builddir:-.}/src/test/test-switch-id" nobody have-caps || exit 1 "${builddir:-.}/src/test/test-switch-id" nobody setuid-keepcaps || exit 1 +if test "$OVERRIDE_GCDA_PERMISSIONS_HACK" = "yes"; then + find src -type f -name '*gcda' -print0 | xargs -0 chmod 0644 +fi echo "All okay" diff --git a/src/test/test_tortls.c b/src/test/test_tortls.c index b9b74a1e96..3a048fb1f0 100644 --- a/src/test/test_tortls.c +++ b/src/test/test_tortls.c @@ -8,19 +8,13 @@ #ifdef _WIN32 #include <winsock2.h> #endif +#include <math.h> -#ifdef __GNUC__ -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -#endif +#include "compat.h" -#if __GNUC__ && GCC_VERSION >= 402 -#if GCC_VERSION >= 406 -#pragma GCC diagnostic push -#endif /* Some versions of OpenSSL declare SSL_get_selected_srtp_profile twice in * srtp.h. Suppress the GCC warning so we can build with -Wredundant-decl. */ -#pragma GCC diagnostic ignored "-Wredundant-decls" -#endif +DISABLE_GCC_WARNING(redundant-decls) #include <openssl/opensslv.h> @@ -33,13 +27,7 @@ #include <openssl/evp.h> #include <openssl/bn.h> -#if __GNUC__ && GCC_VERSION >= 402 -#if GCC_VERSION >= 406 -#pragma GCC diagnostic pop -#else -#pragma GCC diagnostic warning "-Wredundant-decls" -#endif -#endif +ENABLE_GCC_WARNING(redundant-decls) #include "or.h" #include "torlog.h" @@ -50,9 +38,6 @@ #include "log_test_helpers.h" #define NS_MODULE tortls -extern tor_tls_context_t *server_tls_context; -extern tor_tls_context_t *client_tls_context; - #if OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0) \ && !defined(LIBRESSL_VERSION_NUMBER) #define OPENSSL_OPAQUE @@ -277,8 +262,6 @@ test_tortls_get_state_description(void *ignored) tor_free(tls); } -extern int tor_tls_object_ex_data_index; - static void test_tortls_get_by_ssl(void *ignored) { @@ -791,8 +774,6 @@ get_cipher_by_id(uint16_t id) return NULL; } -extern uint16_t v2_cipher_list[]; - static void test_tortls_classify_client_ciphers(void *ignored) { @@ -1185,9 +1166,6 @@ test_tortls_get_forced_write_size(void *ignored) tor_free(tls); } -extern uint64_t total_bytes_written_over_tls; -extern uint64_t total_bytes_written_by_tls; - static void test_tortls_get_write_overhead_ratio(void *ignored) { @@ -1196,17 +1174,17 @@ test_tortls_get_write_overhead_ratio(void *ignored) total_bytes_written_over_tls = 0; ret = tls_get_write_overhead_ratio(); - tt_int_op(ret, OP_EQ, 1.0); + tt_double_op(fabs(ret - 1.0), OP_LT, 1E-12); total_bytes_written_by_tls = 10; total_bytes_written_over_tls = 1; ret = tls_get_write_overhead_ratio(); - tt_int_op(ret, OP_EQ, 10.0); + tt_double_op(fabs(ret - 10.0), OP_LT, 1E-12); total_bytes_written_by_tls = 10; total_bytes_written_over_tls = 2; ret = tls_get_write_overhead_ratio(); - tt_int_op(ret, OP_EQ, 5.0); + tt_double_op(fabs(ret - 5.0), OP_LT, 1E-12); done: (void)0; diff --git a/src/test/test_util.c b/src/test/test_util.c index d534cc0b52..c331a662a5 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -30,6 +30,9 @@ #include <ctype.h> #include <float.h> +#define INFINITY_DBL ((double)INFINITY) +#define NAN_DBL ((double)NAN) + /* XXXX this is a minimal wrapper to make the unit tests compile with the * changed tor_timegm interface. */ static time_t @@ -258,7 +261,7 @@ test_util_time(void *arg) int i; struct timeval tv; - /* Test tv_udiff */ + /* Test tv_udiff and tv_mdiff */ (void)arg; start.tv_sec = 5; @@ -268,22 +271,312 @@ test_util_time(void *arg) end.tv_usec = 5000; tt_int_op(0L,OP_EQ, tv_udiff(&start, &end)); + tt_int_op(0L,OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(0L,OP_EQ, tv_udiff(&end, &start)); + tt_int_op(0L,OP_EQ, tv_mdiff(&end, &start)); end.tv_usec = 7000; tt_int_op(2000L,OP_EQ, tv_udiff(&start, &end)); + tt_int_op(2L,OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(-2000L,OP_EQ, tv_udiff(&end, &start)); + tt_int_op(-2L,OP_EQ, tv_mdiff(&end, &start)); end.tv_sec = 6; tt_int_op(1002000L,OP_EQ, tv_udiff(&start, &end)); + tt_int_op(1002L,OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(-1002000L,OP_EQ, tv_udiff(&end, &start)); + tt_int_op(-1002L,OP_EQ, tv_mdiff(&end, &start)); end.tv_usec = 0; tt_int_op(995000L,OP_EQ, tv_udiff(&start, &end)); + tt_int_op(995L,OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(-995000L,OP_EQ, tv_udiff(&end, &start)); + tt_int_op(-995L,OP_EQ, tv_mdiff(&end, &start)); end.tv_sec = 4; tt_int_op(-1005000L,OP_EQ, tv_udiff(&start, &end)); + tt_int_op(-1005L,OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(1005000L,OP_EQ, tv_udiff(&end, &start)); + tt_int_op(1005L,OP_EQ, tv_mdiff(&end, &start)); + + /* Negative tv_sec values, these will break on platforms where tv_sec is + * unsigned */ + + end.tv_sec = -10; + + tt_int_op(-15005000L,OP_EQ, tv_udiff(&start, &end)); + tt_int_op(-15005L,OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(15005000L,OP_EQ, tv_udiff(&end, &start)); + tt_int_op(15005L,OP_EQ, tv_mdiff(&end, &start)); + + start.tv_sec = -100; + + tt_int_op(89995000L,OP_EQ, tv_udiff(&start, &end)); + tt_int_op(89995L,OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(-89995000L,OP_EQ, tv_udiff(&end, &start)); + tt_int_op(-89995L,OP_EQ, tv_mdiff(&end, &start)); + + /* Test that tv_usec values round away from zero when converted to msec */ + start.tv_sec = 0; + start.tv_usec = 0; + end.tv_sec = 10; + end.tv_usec = 499; + + tt_int_op(10000499L, OP_EQ, tv_udiff(&start, &end)); + tt_int_op(10000L, OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(-10000499L, OP_EQ, tv_udiff(&end, &start)); + tt_int_op(-10000L, OP_EQ, tv_mdiff(&end, &start)); + + start.tv_sec = 0; + start.tv_usec = 0; + end.tv_sec = 10; + end.tv_usec = 500; + + tt_int_op(10000500L, OP_EQ, tv_udiff(&start, &end)); + tt_int_op(10001L, OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(-10000500L, OP_EQ, tv_udiff(&end, &start)); + tt_int_op(-10000L, OP_EQ, tv_mdiff(&end, &start)); + + start.tv_sec = 0; + start.tv_usec = 0; + end.tv_sec = 10; + end.tv_usec = 501; + + tt_int_op(10000501L, OP_EQ, tv_udiff(&start, &end)); + tt_int_op(10001L, OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(-10000501L, OP_EQ, tv_udiff(&end, &start)); + tt_int_op(-10001L, OP_EQ, tv_mdiff(&end, &start)); + + /* Overflow conditions */ + +#ifdef _WIN32 + /* Would you believe that tv_sec is a long on windows? Of course you would.*/ +#define TV_SEC_MAX LONG_MAX +#define TV_SEC_MIN LONG_MIN +#else + /* Some BSDs have struct timeval.tv_sec 64-bit, but time_t (and long) 32-bit + * Which means TIME_MAX is not actually the maximum value of tv_sec. + * But that's ok for the moment, because the code correctly performs 64-bit + * calculations internally, then catches the overflow. */ +#define TV_SEC_MAX TIME_MAX +#define TV_SEC_MIN TIME_MIN +#endif + +/* Assume tv_usec is an unsigned integer until proven otherwise */ +#define TV_USEC_MAX UINT_MAX +#define TOR_USEC_PER_SEC 1000000 + + /* Overflows in the result type */ + + /* All comparisons work */ + start.tv_sec = 0; + start.tv_usec = 0; + end.tv_sec = LONG_MAX/1000 - 2; + end.tv_usec = 0; + + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&start, &end)); + tt_int_op(end.tv_sec*1000L, OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&end, &start)); + tt_int_op(-end.tv_sec*1000L, OP_EQ, tv_mdiff(&end, &start)); + + start.tv_sec = 0; + start.tv_usec = 0; + end.tv_sec = LONG_MAX/1000000 - 1; + end.tv_usec = 0; + + tt_int_op(end.tv_sec*1000000L, OP_EQ, tv_udiff(&start, &end)); + tt_int_op(end.tv_sec*1000L, OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(-end.tv_sec*1000000L, OP_EQ, tv_udiff(&end, &start)); + tt_int_op(-end.tv_sec*1000L, OP_EQ, tv_mdiff(&end, &start)); + + /* No comparisons work */ + start.tv_sec = 0; + start.tv_usec = 0; + end.tv_sec = LONG_MAX/1000 + 1; + end.tv_usec = 0; + + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&end, &start)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&end, &start)); + + start.tv_sec = 0; + start.tv_usec = 0; + end.tv_sec = LONG_MAX/1000000 + 1; + end.tv_usec = 0; + + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&start, &end)); + tt_int_op(end.tv_sec*1000L, OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&end, &start)); + tt_int_op(-end.tv_sec*1000L, OP_EQ, tv_mdiff(&end, &start)); + + start.tv_sec = 0; + start.tv_usec = 0; + end.tv_sec = LONG_MAX/1000; + end.tv_usec = TOR_USEC_PER_SEC; + + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&end, &start)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&end, &start)); + + start.tv_sec = 0; + start.tv_usec = 0; + end.tv_sec = LONG_MAX/1000000; + end.tv_usec = TOR_USEC_PER_SEC; + + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&start, &end)); + tt_int_op((end.tv_sec + 1)*1000L, OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&end, &start)); + tt_int_op(-(end.tv_sec + 1)*1000L, OP_EQ, tv_mdiff(&end, &start)); + + /* Overflows on comparison to zero */ + + start.tv_sec = 0; + start.tv_usec = 0; + + end.tv_sec = TV_SEC_MAX; + end.tv_usec = 0; + + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&end, &start)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&end, &start)); + + end.tv_sec = TV_SEC_MAX; + end.tv_usec = TOR_USEC_PER_SEC; + + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&end, &start)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&end, &start)); + + end.tv_sec = 0; + end.tv_usec = TV_USEC_MAX; + + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&end, &start)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&end, &start)); + + end.tv_sec = TV_SEC_MAX; + end.tv_usec = TV_USEC_MAX; + + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&end, &start)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&end, &start)); + + end.tv_sec = 0; + end.tv_usec = 0; + + start.tv_sec = TV_SEC_MIN; + start.tv_usec = 0; + + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&end, &start)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&end, &start)); + + start.tv_sec = TV_SEC_MIN; + start.tv_usec = TOR_USEC_PER_SEC; + + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&end, &start)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&end, &start)); + + start.tv_sec = TV_SEC_MIN; + start.tv_usec = TV_USEC_MAX; + + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&end, &start)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&end, &start)); + + /* overflows on comparison to maxima / minima */ + + start.tv_sec = TV_SEC_MIN; + start.tv_usec = 0; + + end.tv_sec = TV_SEC_MAX; + end.tv_usec = 0; + + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&end, &start)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&end, &start)); + + end.tv_sec = TV_SEC_MAX; + end.tv_usec = TOR_USEC_PER_SEC; + + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&end, &start)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&end, &start)); + + end.tv_sec = TV_SEC_MAX; + end.tv_usec = 0; + + start.tv_sec = TV_SEC_MIN; + start.tv_usec = 0; + + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&end, &start)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&end, &start)); + + start.tv_sec = TV_SEC_MIN; + start.tv_usec = TOR_USEC_PER_SEC; + + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&end, &start)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&end, &start)); + + /* overflows on comparison to maxima / minima with extra usec */ + + start.tv_sec = TV_SEC_MIN; + start.tv_usec = TOR_USEC_PER_SEC; + + end.tv_sec = TV_SEC_MAX; + end.tv_usec = 0; + + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&end, &start)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&end, &start)); + + end.tv_sec = TV_SEC_MAX; + end.tv_usec = TOR_USEC_PER_SEC; + + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&end, &start)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&end, &start)); + + end.tv_sec = TV_SEC_MAX; + end.tv_usec = TOR_USEC_PER_SEC; + + start.tv_sec = TV_SEC_MIN; + start.tv_usec = 0; + + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&end, &start)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&end, &start)); + + start.tv_sec = TV_SEC_MIN; + start.tv_usec = TOR_USEC_PER_SEC; + + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&start, &end)); + tt_int_op(LONG_MAX, OP_EQ, tv_udiff(&end, &start)); + tt_int_op(LONG_MAX, OP_EQ, tv_mdiff(&end, &start)); /* Test tor_timegm & tor_gmtime_r */ @@ -622,9 +915,16 @@ test_util_time(void *arg) parse_rfc1123_time("Wed, 30 Ene 2011 23:59:59 GMT", &t_res)); tt_int_op(-1,OP_EQ, parse_rfc1123_time("Wed, 30 Mar 2011 23:59:59 GM", &t_res)); + tt_int_op(-1,OP_EQ, + parse_rfc1123_time("Wed, 30 Mar 1900 23:59:59 GMT", &t_res)); + /* Leap year. */ tt_int_op(-1,OP_EQ, parse_rfc1123_time("Wed, 29 Feb 2011 16:00:00 GMT", &t_res)); + tt_int_op(0,OP_EQ, + parse_rfc1123_time("Wed, 29 Feb 2012 16:00:00 GMT", &t_res)); + + /* Leap second plus one */ tt_int_op(-1,OP_EQ, parse_rfc1123_time("Wed, 30 Mar 2011 23:59:61 GMT", &t_res)); @@ -865,106 +1165,106 @@ test_util_config_line(void *arg) , sizeof(buf)); str = buf; - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k"); tt_str_op(v,OP_EQ, "v"); tor_free(k); tor_free(v); tt_assert(!strcmpstart(str, "key value with")); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "key"); tt_str_op(v,OP_EQ, "value with spaces"); tor_free(k); tor_free(v); tt_assert(!strcmpstart(str, "keykey")); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "keykey"); tt_str_op(v,OP_EQ, "val"); tor_free(k); tor_free(v); tt_assert(!strcmpstart(str, "k2\n")); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k2"); tt_str_op(v,OP_EQ, ""); tor_free(k); tor_free(v); tt_assert(!strcmpstart(str, "k3 \n")); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k3"); tt_str_op(v,OP_EQ, ""); tor_free(k); tor_free(v); tt_assert(!strcmpstart(str, "#comment")); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k4"); tt_str_op(v,OP_EQ, ""); tor_free(k); tor_free(v); tt_assert(!strcmpstart(str, "k5#abc")); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k5"); tt_str_op(v,OP_EQ, ""); tor_free(k); tor_free(v); tt_assert(!strcmpstart(str, "k6")); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k6"); tt_str_op(v,OP_EQ, "val"); tor_free(k); tor_free(v); tt_assert(!strcmpstart(str, "kseven")); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "kseven"); tt_str_op(v,OP_EQ, "a quoted \'string"); tor_free(k); tor_free(v); tt_assert(!strcmpstart(str, "k8 ")); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k8"); tt_str_op(v,OP_EQ, "a quoted\n\"str\\ing\t\x01\x01\x01\""); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k9"); tt_str_op(v,OP_EQ, "a line that spans two lines."); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k10"); tt_str_op(v,OP_EQ, "more than one continuation"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k11"); tt_str_op(v,OP_EQ, "continuation at the start"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k12"); tt_str_op(v,OP_EQ, "line with a embedded"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k13"); tt_str_op(v,OP_EQ, "continuation at the very start"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k14"); tt_str_op(v,OP_EQ, "a line that has a comment and" ); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k15"); tt_str_op(v,OP_EQ, "this should be the next new line"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k16"); tt_str_op(v,OP_EQ, "a line that has a comment and" ); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k17"); tt_str_op(v,OP_EQ, "this should be the next new line"); tor_free(k); tor_free(v); @@ -999,32 +1299,36 @@ test_util_config_line_quotes(void *arg) , sizeof(buf4)); str = buf1; - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "kTrailingSpace"); tt_str_op(v,OP_EQ, "quoted value"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_ptr_op(str,OP_EQ, NULL); tor_free(k); tor_free(v); str = buf2; - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_ptr_op(str,OP_EQ, NULL); tor_free(k); tor_free(v); str = buf3; - str = parse_config_line_from_str(str, &k, &v); + const char *err = NULL; + str = parse_config_line_from_str_verbose(str, &k, &v, &err); tt_ptr_op(str,OP_EQ, NULL); tor_free(k); tor_free(v); + tt_str_op(err, OP_EQ, "Invalid escape sequence in quoted string"); str = buf4; - str = parse_config_line_from_str(str, &k, &v); + err = NULL; + str = parse_config_line_from_str_verbose(str, &k, &v, &err); tt_ptr_op(str,OP_EQ, NULL); tor_free(k); tor_free(v); + tt_str_op(err, OP_EQ, "Invalid escape sequence in quoted string"); done: tor_free(k); @@ -1046,12 +1350,12 @@ test_util_config_line_comment_character(void *arg) , sizeof(buf)); str = buf; - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k1"); tt_str_op(v,OP_EQ, "# in quotes"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "k2"); tt_str_op(v,OP_EQ, "some value"); tor_free(k); tor_free(v); @@ -1059,7 +1363,7 @@ test_util_config_line_comment_character(void *arg) tt_str_op(str,OP_EQ, "k3 /home/user/myTorNetwork#2\n"); #if 0 - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); test_streq(k, "k3"); test_streq(v, "/home/user/myTorNetwork#2"); tor_free(k); tor_free(v); @@ -1116,57 +1420,57 @@ test_util_config_line_escaped_content(void *arg) str = buf1; - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "HexadecimalLower"); tt_str_op(v,OP_EQ, "*"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "HexadecimalUpper"); tt_str_op(v,OP_EQ, "*"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "HexadecimalUpperX"); tt_str_op(v,OP_EQ, "*"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "Octal"); tt_str_op(v,OP_EQ, "*"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "Newline"); tt_str_op(v,OP_EQ, "\n"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "Tab"); tt_str_op(v,OP_EQ, "\t"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "CarriageReturn"); tt_str_op(v,OP_EQ, "\r"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "DoubleQuote"); tt_str_op(v,OP_EQ, "\""); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "SimpleQuote"); tt_str_op(v,OP_EQ, "'"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "Backslash"); tt_str_op(v,OP_EQ, "\\"); tor_free(k); tor_free(v); - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_str_op(k,OP_EQ, "Mix"); tt_str_op(v,OP_EQ, "This is a \"star\":\t'*'\nAnd second line"); tor_free(k); tor_free(v); @@ -1174,36 +1478,81 @@ test_util_config_line_escaped_content(void *arg) str = buf2; - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_ptr_op(str,OP_EQ, NULL); tor_free(k); tor_free(v); str = buf3; - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_ptr_op(str,OP_EQ, NULL); tor_free(k); tor_free(v); str = buf4; - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_ptr_op(str,OP_EQ, NULL); tor_free(k); tor_free(v); #if 0 str = buf5; - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_ptr_op(str, OP_EQ, NULL); tor_free(k); tor_free(v); #endif str = buf6; - str = parse_config_line_from_str(str, &k, &v); + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); + tt_ptr_op(str,OP_EQ, NULL); + tor_free(k); tor_free(v); + + /* more things to try. */ + /* Bad hex: */ + strlcpy(buf1, "Foo \"\\x9g\"\n", sizeof(buf1)); + strlcpy(buf2, "Foo \"\\xg0\"\n", sizeof(buf2)); + strlcpy(buf3, "Foo \"\\xf\"\n", sizeof(buf3)); + /* bad escape */ + strlcpy(buf4, "Foo \"\\q\"\n", sizeof(buf4)); + /* missing endquote */ + strlcpy(buf5, "Foo \"hello\n", sizeof(buf5)); + /* extra stuff */ + strlcpy(buf6, "Foo \"hello\" world\n", sizeof(buf6)); + + str=buf1; + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); tt_ptr_op(str,OP_EQ, NULL); tor_free(k); tor_free(v); + str=buf2; + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); + tt_ptr_op(str,OP_EQ, NULL); + tor_free(k); tor_free(v); + + str=buf3; + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); + tt_ptr_op(str,OP_EQ, NULL); + tor_free(k); tor_free(v); + + str=buf4; + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); + tt_ptr_op(str,OP_EQ, NULL); + tor_free(k); tor_free(v); + + str=buf5; + + str = parse_config_line_from_str_verbose(str, &k, &v, NULL); + tt_ptr_op(str,OP_EQ, NULL); + tor_free(k); tor_free(v); + + str=buf6; + const char *err = NULL; + str = parse_config_line_from_str_verbose(str, &k, &v, &err); + tt_ptr_op(str,OP_EQ, NULL); + tor_free(k); tor_free(v); + tt_str_op(err,OP_EQ, "Excess data after quoted string"); + done: tor_free(k); tor_free(v); @@ -1428,6 +1777,11 @@ test_util_strmisc(void *arg) tt_int_op(-50L,OP_EQ, tor_parse_long("-50 plus garbage",10,-100,100,&i,&cp)); tt_int_op(1,OP_EQ, i); tt_str_op(cp,OP_EQ, " plus garbage"); + /* Illogical min max */ + tt_int_op(0L,OP_EQ, tor_parse_long("10",10,50,4,&i,NULL)); + tt_int_op(0,OP_EQ, i); + tt_int_op(0L,OP_EQ, tor_parse_long("-50",10,100,-100,&i,NULL)); + tt_int_op(0,OP_EQ, i); /* Out of bounds */ tt_int_op(0L,OP_EQ, tor_parse_long("10",10,50,100,&i,NULL)); tt_int_op(0,OP_EQ, i); @@ -1472,24 +1826,24 @@ test_util_strmisc(void *arg) { /* Test parse_double */ - double d = tor_parse_double("10", 0, UINT64_MAX,&i,NULL); + double d = tor_parse_double("10", 0, (double)UINT64_MAX,&i,NULL); tt_int_op(1,OP_EQ, i); tt_assert(DBL_TO_U64(d) == 10); - d = tor_parse_double("0", 0, UINT64_MAX,&i,NULL); + d = tor_parse_double("0", 0, (double)UINT64_MAX,&i,NULL); tt_int_op(1,OP_EQ, i); tt_assert(DBL_TO_U64(d) == 0); - d = tor_parse_double(" ", 0, UINT64_MAX,&i,NULL); + d = tor_parse_double(" ", 0, (double)UINT64_MAX,&i,NULL); tt_int_op(0,OP_EQ, i); - d = tor_parse_double(".0a", 0, UINT64_MAX,&i,NULL); + d = tor_parse_double(".0a", 0, (double)UINT64_MAX,&i,NULL); tt_int_op(0,OP_EQ, i); - d = tor_parse_double(".0a", 0, UINT64_MAX,&i,&cp); + d = tor_parse_double(".0a", 0, (double)UINT64_MAX,&i,&cp); tt_int_op(1,OP_EQ, i); - d = tor_parse_double("-.0", 0, UINT64_MAX,&i,NULL); + d = tor_parse_double("-.0", 0, (double)UINT64_MAX,&i,NULL); tt_int_op(1,OP_EQ, i); tt_assert(DBL_TO_U64(d) == 0); d = tor_parse_double("-10", -100.0, 100.0,&i,NULL); tt_int_op(1,OP_EQ, i); - tt_int_op(-10.0,OP_EQ, d); + tt_double_op(fabs(d - -10.0),OP_LT, 1E-12); } { @@ -1583,6 +1937,17 @@ test_util_strmisc(void *arg) tt_str_op("\"z\\001abc\\277d\"",OP_EQ, escaped("z\001abc\277d")); tt_str_op("\"z\\336\\255 ;foo\"",OP_EQ, escaped("z\xde\xad\x20;foo")); + /* Other cases of esc_for_log{,_len} */ + cp_tmp = esc_for_log(NULL); + tt_str_op(cp_tmp, OP_EQ, "(null)"); + tor_free(cp_tmp); + cp_tmp = esc_for_log_len("abcdefg", 3); + tt_str_op(cp_tmp, OP_EQ, "\"abc\""); + tor_free(cp_tmp); + cp_tmp = esc_for_log_len("abcdefg", 100); + tt_str_op(cp_tmp, OP_EQ, "\"abcdefg\""); + tor_free(cp_tmp); + /* Test strndup and memdup */ { const char *s = "abcdefghijklmnopqrstuvwxyz"; @@ -1737,22 +2102,21 @@ test_util_gzip(void *arg) (void)arg; buf1 = tor_strdup("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAZAAAAAAAAAAAAAAAAAAAZ"); tt_assert(detect_compression_method(buf1, strlen(buf1)) == UNKNOWN_METHOD); - if (is_gzip_supported()) { - tt_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, - GZIP_METHOD)); - tt_assert(buf2); - tt_assert(len1 < strlen(buf1)); - tt_assert(detect_compression_method(buf2, len1) == GZIP_METHOD); - - tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, - GZIP_METHOD, 1, LOG_INFO)); - tt_assert(buf3); - tt_int_op(strlen(buf1) + 1,OP_EQ, len2); - tt_str_op(buf1,OP_EQ, buf3); - - tor_free(buf2); - tor_free(buf3); - } + + tt_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, + GZIP_METHOD)); + tt_assert(buf2); + tt_assert(len1 < strlen(buf1)); + tt_assert(detect_compression_method(buf2, len1) == GZIP_METHOD); + + tt_assert(!tor_gzip_uncompress(&buf3, &len2, buf2, len1, + GZIP_METHOD, 1, LOG_INFO)); + tt_assert(buf3); + tt_int_op(strlen(buf1) + 1,OP_EQ, len2); + tt_str_op(buf1,OP_EQ, buf3); + + tor_free(buf2); + tor_free(buf3); tt_assert(!tor_gzip_compress(&buf2, &len1, buf1, strlen(buf1)+1, ZLIB_METHOD)); @@ -1836,6 +2200,52 @@ test_util_gzip(void *arg) tor_free(buf1); } +static void +test_util_gzip_compression_bomb(void *arg) +{ + /* A 'compression bomb' is a very small object that uncompresses to a huge + * one. Most compression formats support them, but they can be a DOS vector. + * In Tor we try not to generate them, and we don't accept them. + */ + (void) arg; + size_t one_million = 1<<20; + char *one_mb = tor_malloc_zero(one_million); + char *result = NULL; + size_t result_len = 0; + tor_zlib_state_t *state = NULL; + + /* Make sure we can't produce a compression bomb */ + tt_int_op(-1, OP_EQ, tor_gzip_compress(&result, &result_len, + one_mb, one_million, + ZLIB_METHOD)); + + /* Here's a compression bomb that we made manually. */ + const char compression_bomb[1039] = + { 0x78, 0xDA, 0xED, 0xC1, 0x31, 0x01, 0x00, 0x00, 0x00, 0xC2, + 0xA0, 0xF5, 0x4F, 0x6D, 0x08, 0x5F, 0xA0 /* .... */ }; + tt_int_op(-1, OP_EQ, tor_gzip_uncompress(&result, &result_len, + compression_bomb, 1039, + ZLIB_METHOD, 0, LOG_WARN)); + + /* Now try streaming that. */ + state = tor_zlib_new(0, ZLIB_METHOD, HIGH_COMPRESSION); + tor_zlib_output_t r; + const char *inp = compression_bomb; + size_t inlen = 1039; + do { + char *outp = one_mb; + size_t outleft = 4096; /* small on purpose */ + r = tor_zlib_process(state, &outp, &outleft, &inp, &inlen, 0); + tt_int_op(inlen, OP_NE, 0); + } while (r == TOR_ZLIB_BUF_FULL); + + tt_int_op(r, OP_EQ, TOR_ZLIB_ERR); + + done: + tor_free(one_mb); + tor_zlib_free(state); +} + /** Run unit tests for mmap() wrapper functionality. */ static void test_util_mmap(void *arg) @@ -2842,19 +3252,40 @@ test_util_memarea(void *arg) p1 = memarea_alloc(area, 1); tt_ptr_op(p1,OP_EQ, p1_orig); memarea_clear(area); + size_t total = 0, initial_allocation, allocation2, dummy; + memarea_get_stats(area, &initial_allocation, &dummy); /* Check for running over an area's size. */ - for (i = 0; i < 512; ++i) { - p1 = memarea_alloc(area, crypto_rand_int(5)+1); + for (i = 0; i < 4096; ++i) { + size_t n = crypto_rand_int(6); + p1 = memarea_alloc(area, n); + total += n; tt_assert(memarea_owns_ptr(area, p1)); } memarea_assert_ok(area); + memarea_get_stats(area, &allocation2, &dummy); /* Make sure we can allocate a too-big object. */ p1 = memarea_alloc_zero(area, 9000); p2 = memarea_alloc_zero(area, 16); + total += 9000; + total += 16; tt_assert(memarea_owns_ptr(area, p1)); tt_assert(memarea_owns_ptr(area, p2)); + /* Now test stats... */ + size_t allocated = 0, used = 0; + memarea_get_stats(area, &allocated, &used); + tt_int_op(used, OP_LE, allocated); + tt_int_op(used, OP_GE, total); /* not EQ, because of alignment and headers*/ + tt_int_op(allocated, OP_GT, allocation2); + + tt_int_op(allocation2, OP_GT, initial_allocation); + + memarea_clear(area); + memarea_get_stats(area, &allocated, &used); + tt_int_op(used, OP_LT, 128); /* Not 0, because of header */ + tt_int_op(allocated, OP_EQ, initial_allocation); + done: memarea_drop_all(area); tor_free(malloced_ptr); @@ -3244,6 +3675,21 @@ test_util_ftruncate(void *ptr) tor_free(buf); } +static void +test_util_num_cpus(void *arg) +{ + (void)arg; + int num = compute_num_cpus(); + if (num < 0) + tt_skip(); + + tt_int_op(num, OP_GE, 1); + tt_int_op(num, OP_LE, 16); + + done: + ; +} + #ifdef _WIN32 static void test_util_load_win_lib(void *ptr) @@ -4223,21 +4669,6 @@ test_util_round_to_next_multiple_of(void *arg) tt_u64_op(round_uint64_to_next_multiple_of(UINT64_MAX,2), ==, UINT64_MAX); - tt_i64_op(round_int64_to_next_multiple_of(0,1), ==, 0); - tt_i64_op(round_int64_to_next_multiple_of(0,7), ==, 0); - - tt_i64_op(round_int64_to_next_multiple_of(99,1), ==, 99); - tt_i64_op(round_int64_to_next_multiple_of(99,7), ==, 105); - tt_i64_op(round_int64_to_next_multiple_of(99,9), ==, 99); - - tt_i64_op(round_int64_to_next_multiple_of(-99,1), ==, -99); - tt_i64_op(round_int64_to_next_multiple_of(-99,7), ==, -98); - tt_i64_op(round_int64_to_next_multiple_of(-99,9), ==, -99); - - tt_i64_op(round_int64_to_next_multiple_of(INT64_MIN,2), ==, INT64_MIN); - tt_i64_op(round_int64_to_next_multiple_of(INT64_MAX,2), ==, - INT64_MAX); - tt_int_op(round_uint32_to_next_multiple_of(0,1), ==, 0); tt_int_op(round_uint32_to_next_multiple_of(0,7), ==, 0); @@ -4407,7 +4838,7 @@ test_util_clamp_double_to_int64(void *arg) { (void)arg; - tt_i64_op(INT64_MIN, ==, clamp_double_to_int64(-INFINITY)); + tt_i64_op(INT64_MIN, ==, clamp_double_to_int64(-INFINITY_DBL)); tt_i64_op(INT64_MIN, ==, clamp_double_to_int64(-1.0 * pow(2.0, 64.0) - 1.0)); tt_i64_op(INT64_MIN, ==, @@ -4420,7 +4851,7 @@ test_util_clamp_double_to_int64(void *arg) tt_i64_op(0, ==, clamp_double_to_int64(-0.9)); tt_i64_op(0, ==, clamp_double_to_int64(-0.1)); tt_i64_op(0, ==, clamp_double_to_int64(0.0)); - tt_i64_op(0, ==, clamp_double_to_int64(NAN)); + tt_i64_op(0, ==, clamp_double_to_int64(NAN_DBL)); tt_i64_op(0, ==, clamp_double_to_int64(0.1)); tt_i64_op(0, ==, clamp_double_to_int64(0.9)); tt_i64_op(1, ==, clamp_double_to_int64(1.0)); @@ -4432,7 +4863,7 @@ test_util_clamp_double_to_int64(void *arg) clamp_double_to_int64(pow(2.0, 63.0))); tt_i64_op(INT64_MAX, ==, clamp_double_to_int64(pow(2.0, 64.0))); - tt_i64_op(INT64_MAX, ==, clamp_double_to_int64(INFINITY)); + tt_i64_op(INT64_MAX, ==, clamp_double_to_int64(INFINITY_DBL)); done: ; @@ -4780,12 +5211,71 @@ test_util_pwdb(void *arg) dir = get_user_homedir(name); tt_assert(dir != NULL); + /* Try failing cases. First find a user that doesn't exist by name */ + char rand[4]; + char badname[9]; + int i, found=0; + for (i = 0; i < 100; ++i) { + crypto_rand(rand, sizeof(rand)); + base16_encode(badname, sizeof(badname), rand, sizeof(rand)); + if (tor_getpwnam(badname) == NULL) { + found = 1; + break; + } + } + tt_assert(found); + tor_free(dir); + dir = get_user_homedir(badname); + tt_assert(dir == NULL); + + /* Now try to find a user that doesn't exist by ID. */ + found = 0; + for (i = 0; i < 1000; ++i) { + uid_t u; + crypto_rand((char*)&u, sizeof(u)); + if (tor_getpwuid(u) == NULL) { + found = 1; + break; + } + } + tt_assert(found); + done: tor_free(name); tor_free(dir); } #endif +static void +test_util_calloc_check(void *arg) +{ + (void) arg; + /* Easy cases that are good. */ + tt_assert(size_mul_check__(0,0)); + tt_assert(size_mul_check__(0,100)); + tt_assert(size_mul_check__(100,0)); + tt_assert(size_mul_check__(100,100)); + + /* Harder cases that are still good. */ + tt_assert(size_mul_check__(SIZE_MAX, 1)); + tt_assert(size_mul_check__(1, SIZE_MAX)); + tt_assert(size_mul_check__(SIZE_MAX / 10, 9)); + tt_assert(size_mul_check__(11, SIZE_MAX / 12)); + const size_t sqrt_size_max_p1 = ((size_t)1) << (sizeof(size_t) * 4); + tt_assert(size_mul_check__(sqrt_size_max_p1, sqrt_size_max_p1 - 1)); + + /* Cases that overflow */ + tt_assert(! size_mul_check__(SIZE_MAX, 2)); + tt_assert(! size_mul_check__(2, SIZE_MAX)); + tt_assert(! size_mul_check__(SIZE_MAX / 10, 11)); + tt_assert(! size_mul_check__(11, SIZE_MAX / 10)); + tt_assert(! size_mul_check__(SIZE_MAX / 8, 9)); + tt_assert(! size_mul_check__(sqrt_size_max_p1, sqrt_size_max_p1)); + + done: + ; +} + #define UTIL_LEGACY(name) \ { #name, test_util_ ## name , 0, NULL, NULL } @@ -4815,11 +5305,12 @@ struct testcase_t util_tests[] = { UTIL_LEGACY(strmisc), UTIL_LEGACY(pow2), UTIL_LEGACY(gzip), + UTIL_LEGACY(gzip_compression_bomb), UTIL_LEGACY(datadir), UTIL_LEGACY(memarea), UTIL_LEGACY(control_formats), UTIL_LEGACY(mmap), - UTIL_LEGACY(sscanf), + UTIL_TEST(sscanf, TT_FORK), UTIL_LEGACY(format_time_interval), UTIL_LEGACY(path_is_relative), UTIL_LEGACY(strtok), @@ -4834,6 +5325,7 @@ struct testcase_t util_tests[] = { UTIL_TEST(listdir, 0), UTIL_TEST(parent_dir, 0), UTIL_TEST(ftruncate, 0), + UTIL_TEST(num_cpus, 0), UTIL_TEST_WIN_ONLY(load_win_lib, 0), UTIL_TEST_NO_WIN(exit_status, 0), UTIL_TEST_NO_WIN(fgets_eagain, 0), @@ -4871,6 +5363,7 @@ struct testcase_t util_tests[] = { UTIL_TEST(get_avail_disk_space, 0), UTIL_TEST(touch_file, 0), UTIL_TEST_NO_WIN(pwdb, TT_FORK), + UTIL_TEST(calloc_check, 0), END_OF_TESTCASES }; diff --git a/src/test/test_util_format.c b/src/test/test_util_format.c index a25054cd0a..f3dcd1184c 100644 --- a/src/test/test_util_format.c +++ b/src/test/test_util_format.c @@ -263,14 +263,14 @@ test_util_format_base16_decode(void *ignored) res = base16_decode(dst, 1, src, 10); tt_int_op(res, OP_EQ, -1); - res = base16_decode(dst, SIZE_T_CEILING+2, src, 10); + res = base16_decode(dst, ((size_t)INT_MAX)+1, src, 10); tt_int_op(res, OP_EQ, -1); res = base16_decode(dst, 1000, "", 0); tt_int_op(res, OP_EQ, 0); res = base16_decode(dst, 1000, "aabc", 4); - tt_int_op(res, OP_EQ, 0); + tt_int_op(res, OP_EQ, 2); tt_mem_op(dst, OP_EQ, "\xaa\xbc", 2); res = base16_decode(dst, 1000, "aabcd", 6); @@ -280,7 +280,7 @@ test_util_format_base16_decode(void *ignored) tt_int_op(res, OP_EQ, -1); res = base16_decode(real_dst, 10, real_src, 14); - tt_int_op(res, OP_EQ, 0); + tt_int_op(res, OP_EQ, 7); tt_mem_op(real_dst, OP_EQ, expected, 7); done: @@ -289,6 +289,95 @@ test_util_format_base16_decode(void *ignored) tor_free(real_dst); } +static void +test_util_format_base32_encode(void *arg) +{ + (void) arg; + size_t real_dstlen = 32; + char *dst = tor_malloc_zero(real_dstlen); + + /* Basic use case that doesn't require a source length correction. */ + { + /* Length of 10 bytes. */ + const char *src = "blahbleh12"; + size_t srclen = strlen(src); + /* Expected result encoded base32. This was created using python as + * such (and same goes for all test case.): + * + * b = bytes("blahbleh12", 'utf-8') + * base64.b32encode(b) + * (result in lower case) + */ + const char *expected = "mjwgc2dcnrswqmjs"; + + base32_encode(dst, base32_encoded_size(srclen), src, srclen); + tt_mem_op(expected, OP_EQ, dst, strlen(expected)); + /* Encode but to a larger size destination. */ + memset(dst, 0, real_dstlen); + base32_encode(dst, real_dstlen, src, srclen); + tt_mem_op(expected, OP_EQ, dst, strlen(expected)); + } + + /* Non multiple of 5 for the source buffer length. */ + { + /* Length of 8 bytes. */ + const char *expected = "mjwgc2dcnrswq"; + const char *src = "blahbleh"; + size_t srclen = strlen(src); + + memset(dst, 0, real_dstlen); + base32_encode(dst, base32_encoded_size(srclen), src, srclen); + tt_mem_op(expected, OP_EQ, dst, strlen(expected)); + } + + done: + tor_free(dst); +} + +static void +test_util_format_base32_decode(void *arg) +{ + (void) arg; + int ret; + size_t real_dstlen = 32; + char *dst = tor_malloc_zero(real_dstlen); + + /* Basic use case. */ + { + /* Length of 10 bytes. */ + const char *expected = "blahbleh12"; + /* Expected result encoded base32. */ + const char *src = "mjwgc2dcnrswqmjs"; + + ret = base32_decode(dst, strlen(expected), src, strlen(src)); + tt_int_op(ret, ==, 0); + tt_str_op(expected, OP_EQ, dst); + } + + /* Non multiple of 5 for the source buffer length. */ + { + /* Length of 8 bytes. */ + const char *expected = "blahbleh"; + const char *src = "mjwgc2dcnrswq"; + + ret = base32_decode(dst, strlen(expected), src, strlen(src)); + tt_int_op(ret, ==, 0); + tt_mem_op(expected, OP_EQ, dst, strlen(expected)); + } + + /* Invalid values. */ + { + /* Invalid character '#'. */ + ret = base32_decode(dst, real_dstlen, "#abcde", 6); + tt_int_op(ret, ==, -1); + /* Make sure the destination buffer has been zeroed even on error. */ + tt_int_op(tor_mem_is_zero(dst, real_dstlen), ==, 1); + } + + done: + tor_free(dst); +} + struct testcase_t util_format_tests[] = { { "unaligned_accessors", test_util_format_unaligned_accessors, 0, NULL, NULL }, @@ -297,6 +386,10 @@ struct testcase_t util_format_tests[] = { NULL, NULL }, { "base64_decode", test_util_format_base64_decode, 0, NULL, NULL }, { "base16_decode", test_util_format_base16_decode, 0, NULL, NULL }, + { "base32_encode", test_util_format_base32_encode, 0, + NULL, NULL }, + { "base32_decode", test_util_format_base32_decode, 0, + NULL, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_workqueue.c b/src/test/test_workqueue.c index cbcf596b22..75d68a7ce0 100644 --- a/src/test/test_workqueue.c +++ b/src/test/test_workqueue.c @@ -400,6 +400,9 @@ main(int argc, char **argv) } rq = replyqueue_new(as_flags); + if (as_flags && rq == NULL) + return 77; // 77 means "skipped". + tor_assert(rq); tp = threadpool_new(opt_n_threads, rq, new_state, free_state, NULL); diff --git a/src/test/test_workqueue_cancel.sh b/src/test/test_workqueue_cancel.sh new file mode 100755 index 0000000000..f7c663171e --- /dev/null +++ b/src/test/test_workqueue_cancel.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +${builddir:-.}/src/test/test_workqueue -C 1 + diff --git a/src/test/test_workqueue_efd.sh b/src/test/test_workqueue_efd.sh new file mode 100755 index 0000000000..4d89396819 --- /dev/null +++ b/src/test/test_workqueue_efd.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +${builddir:-.}/src/test/test_workqueue \ + --no-eventfd2 --no-pipe2 --no-pipe --no-socketpair diff --git a/src/test/test_workqueue_efd2.sh b/src/test/test_workqueue_efd2.sh new file mode 100755 index 0000000000..7cfff45ff3 --- /dev/null +++ b/src/test/test_workqueue_efd2.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +${builddir:-.}/src/test/test_workqueue \ + --no-eventfd --no-pipe2 --no-pipe --no-socketpair diff --git a/src/test/test_workqueue_pipe.sh b/src/test/test_workqueue_pipe.sh new file mode 100755 index 0000000000..afcef87853 --- /dev/null +++ b/src/test/test_workqueue_pipe.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +${builddir:-.}/src/test/test_workqueue \ + --no-eventfd2 --no-eventfd --no-pipe2 --no-socketpair diff --git a/src/test/test_workqueue_pipe2.sh b/src/test/test_workqueue_pipe2.sh new file mode 100755 index 0000000000..a20a1427e0 --- /dev/null +++ b/src/test/test_workqueue_pipe2.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +${builddir:-.}/src/test/test_workqueue \ + --no-eventfd2 --no-eventfd --no-pipe --no-socketpair diff --git a/src/test/test_workqueue_socketpair.sh b/src/test/test_workqueue_socketpair.sh new file mode 100755 index 0000000000..76af79746d --- /dev/null +++ b/src/test/test_workqueue_socketpair.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +${builddir:-.}/src/test/test_workqueue \ + --no-eventfd2 --no-eventfd --no-pipe2 --no-pipe diff --git a/src/test/testing_common.c b/src/test/testing_common.c index 39c3d02ab1..ea9366305c 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -3,6 +3,8 @@ * Copyright (c) 2007-2016, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +extern const char tor_git_revision[]; + /* Ordinarily defined in tor_main.c; this bit is just here to provide one * since we're not linking to tor_main.c */ const char tor_git_revision[] = ""; @@ -215,8 +217,6 @@ const struct testcase_setup_t passthrough_setup = { passthrough_test_setup, passthrough_test_cleanup }; -extern struct testgroup_t testgroups[]; - /** Main entry point for unit test code: parse the command line, and run * some unit tests. */ int diff --git a/src/test/vote_descriptors.inc b/src/test/vote_descriptors.inc index c5ce21f744..895dc6c65c 100644 --- a/src/test/vote_descriptors.inc +++ b/src/test/vote_descriptors.inc @@ -1,4 +1,4 @@ -const char* VOTE_BODY_V3 = +static const char* VOTE_BODY_V3 = "network-status-version 3\n" "vote-status vote\n" "consensus-methods 13 14 15 16 17 18 19 20 21\n" diff --git a/src/tools/include.am b/src/tools/include.am index 38ed57546f..d0185b5887 100644 --- a/src/tools/include.am +++ b/src/tools/include.am @@ -7,23 +7,27 @@ endif src_tools_tor_resolve_SOURCES = src/tools/tor-resolve.c src_tools_tor_resolve_LDFLAGS = -src_tools_tor_resolve_LDADD = src/common/libor.a @TOR_LIB_MATH@ @TOR_LIB_WS32@ +src_tools_tor_resolve_LDADD = src/common/libor.a \ + src/common/libor-ctime.a \ + @TOR_LIB_MATH@ @TOR_LIB_WS32@ if COVERAGE_ENABLED src_tools_tor_cov_resolve_SOURCES = src/tools/tor-resolve.c src_tools_tor_cov_resolve_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_tools_tor_cov_resolve_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) src_tools_tor_cov_resolve_LDADD = src/common/libor-testing.a \ - @TOR_LIB_MATH@ @TOR_LIB_WS32@ + src/common/libor-ctime-testing.a \ + @TOR_LIB_MATH@ @TOR_LIB_WS32@ endif src_tools_tor_gencert_SOURCES = src/tools/tor-gencert.c src_tools_tor_gencert_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ src_tools_tor_gencert_LDADD = src/common/libor.a src/common/libor-crypto.a \ + src/common/libor-ctime.a \ $(LIBKECCAK_TINY) \ $(LIBDONNA) \ - @TOR_LIB_MATH@ @TOR_ZLIB_LIBS@ @TOR_OPENSSL_LIBS@ \ - @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ + @TOR_LIB_MATH@ @TOR_ZLIB_LIBS@ @TOR_OPENSSL_LIBS@ \ + @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ if COVERAGE_ENABLED src_tools_tor_cov_gencert_SOURCES = src/tools/tor-gencert.c @@ -32,18 +36,21 @@ src_tools_tor_cov_gencert_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) src_tools_tor_cov_gencert_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ src_tools_tor_cov_gencert_LDADD = src/common/libor-testing.a \ src/common/libor-crypto-testing.a \ + src/common/libor-ctime-testing.a \ $(LIBKECCAK_TINY) \ $(LIBDONNA) \ - @TOR_LIB_MATH@ @TOR_ZLIB_LIBS@ @TOR_OPENSSL_LIBS@ \ - @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ + @TOR_LIB_MATH@ @TOR_ZLIB_LIBS@ @TOR_OPENSSL_LIBS@ \ + @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ endif src_tools_tor_checkkey_SOURCES = src/tools/tor-checkkey.c src_tools_tor_checkkey_LDFLAGS = @TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ -src_tools_tor_checkkey_LDADD = src/common/libor.a src/common/libor-crypto.a \ +src_tools_tor_checkkey_LDADD = src/common/libor.a \ + src/common/libor-ctime.a \ + src/common/libor-crypto.a \ $(LIBKECCAK_TINY) \ $(LIBDONNA) \ - @TOR_LIB_MATH@ @TOR_ZLIB_LIBS@ @TOR_OPENSSL_LIBS@ \ - @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ + @TOR_LIB_MATH@ @TOR_ZLIB_LIBS@ @TOR_OPENSSL_LIBS@ \ + @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ EXTRA_DIST += src/tools/tor-fw-helper/README diff --git a/src/tools/tor-gencert.c b/src/tools/tor-gencert.c index 5f2cd3a92d..4ddfbc9657 100644 --- a/src/tools/tor-gencert.c +++ b/src/tools/tor-gencert.c @@ -13,19 +13,12 @@ #include <unistd.h> #endif -#ifdef __GNUC__ -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) -#endif +#include "compat.h" -#if __GNUC__ && GCC_VERSION >= 402 -#if GCC_VERSION >= 406 -#pragma GCC diagnostic push -#endif /* Some versions of OpenSSL declare X509_STORE_CTX_set_verify_cb twice in * x509.h and x509_vfy.h. Suppress the GCC warning so we can build with * -Wredundant-decl. */ -#pragma GCC diagnostic ignored "-Wredundant-decls" -#endif +DISABLE_GCC_WARNING(redundant-decls) #include <openssl/evp.h> #include <openssl/pem.h> @@ -34,13 +27,7 @@ #include <openssl/obj_mac.h> #include <openssl/err.h> -#if __GNUC__ && GCC_VERSION >= 402 -#if GCC_VERSION >= 406 -#pragma GCC diagnostic pop -#else -#pragma GCC diagnostic warning "-Wredundant-decls" -#endif -#endif +ENABLE_GCC_WARNING(redundant-decls) #include <errno.h> #if 0 @@ -61,21 +48,21 @@ #define DEFAULT_LIFETIME 12 /* These globals are set via command line options. */ -char *identity_key_file = NULL; -char *signing_key_file = NULL; -char *certificate_file = NULL; -int reuse_signing_key = 0; -int verbose = 0; -int make_new_id = 0; -int months_lifetime = DEFAULT_LIFETIME; -int passphrase_fd = -1; -char *address = NULL; - -char *passphrase = NULL; -size_t passphrase_len = 0; - -EVP_PKEY *identity_key = NULL; -EVP_PKEY *signing_key = NULL; +static char *identity_key_file = NULL; +static char *signing_key_file = NULL; +static char *certificate_file = NULL; +static int reuse_signing_key = 0; +static int verbose = 0; +static int make_new_id = 0; +static int months_lifetime = DEFAULT_LIFETIME; +static int passphrase_fd = -1; +static char *address = NULL; + +static char *passphrase = NULL; +static size_t passphrase_len = 0; + +static EVP_PKEY *identity_key = NULL; +static EVP_PKEY *signing_key = NULL; /** Write a usage message for tor-gencert to stderr. */ static void diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 610ef2f611..9469245ba7 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -229,7 +229,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.2.8.4-rc-dev" +#define VERSION "0.2.9.0-alpha-dev" |