diff options
Diffstat (limited to 'src')
277 files changed, 13754 insertions, 7736 deletions
diff --git a/src/common/address.c b/src/common/address.c index 5074c1ccf0..9446675712 100644 --- a/src/common/address.c +++ b/src/common/address.c @@ -1700,7 +1700,7 @@ get_interface_address6_via_udp_socket_hack,(int severity, sa_family_t family, tor_addr_t *addr)) { - struct sockaddr_storage my_addr, target_addr; + struct sockaddr_storage target_addr; int sock=-1, r=-1; socklen_t addr_len; @@ -1743,21 +1743,19 @@ get_interface_address6_via_udp_socket_hack,(int severity, goto err; } - if (tor_getsockname(sock,(struct sockaddr*)&my_addr, &addr_len)) { + if (tor_addr_from_getsockname(addr, sock) < 0) { int e = tor_socket_errno(sock); log_fn(severity, LD_NET, "getsockname() to determine interface failed: %s", tor_socket_strerror(e)); goto err; } - if (tor_addr_from_sockaddr(addr, (struct sockaddr*)&my_addr, NULL) == 0) { - if (tor_addr_is_loopback(addr) || tor_addr_is_multicast(addr)) { - log_fn(severity, LD_NET, "Address that we determined via UDP socket" - " magic is unsuitable for public comms."); - } else { - r=0; - } - } + if (tor_addr_is_loopback(addr) || tor_addr_is_multicast(addr)) { + log_fn(severity, LD_NET, "Address that we determined via UDP socket" + " magic is unsuitable for public comms."); + } else { + r=0; + } err: if (sock >= 0) diff --git a/src/common/address_set.c b/src/common/address_set.c index f61fa294e0..b2f4bb4c95 100644 --- a/src/common/address_set.c +++ b/src/common/address_set.c @@ -15,7 +15,7 @@ #include "address.h" #include "compat.h" #include "container.h" -#include "crypto.h" +#include "crypto_rand.h" #include "util.h" #include "siphash.h" diff --git a/src/common/aes.c b/src/common/aes.c index 5d0841dfa3..49bb54762f 100644 --- a/src/common/aes.c +++ b/src/common/aes.c @@ -16,8 +16,8 @@ #include <ws2tcpip.h> #endif +#include "compat_openssl.h" #include <openssl/opensslv.h> -#include "crypto.h" #include "crypto_openssl_mgt.h" #if OPENSSL_VERSION_NUMBER < OPENSSL_V_SERIES(1,0,0) @@ -41,6 +41,7 @@ ENABLE_GCC_WARNING(redundant-decls) #include "util.h" #include "torlog.h" #include "di_ops.h" +#include "crypto_util.h" #ifdef ANDROID /* Android's OpenSSL seems to have removed all of its Engine support. */ @@ -116,7 +117,11 @@ aes_cipher_free_(aes_cnt_cipher_t *cipher_) if (!cipher_) return; EVP_CIPHER_CTX *cipher = (EVP_CIPHER_CTX *) cipher_; +#ifdef OPENSSL_1_1_API + EVP_CIPHER_CTX_reset(cipher); +#else EVP_CIPHER_CTX_cleanup(cipher); +#endif EVP_CIPHER_CTX_free(cipher); } void @@ -403,4 +408,3 @@ aes_set_iv(aes_cnt_cipher_t *cipher, const uint8_t *iv) } #endif /* defined(USE_EVP_AES_CTR) */ - diff --git a/src/common/buffers.h b/src/common/buffers.h index 22a5f7bfa3..4275152de2 100644 --- a/src/common/buffers.h +++ b/src/common/buffers.h @@ -13,7 +13,6 @@ #define TOR_BUFFERS_H #include "compat.h" -#include "compat.h" #include "torint.h" #include "testsupport.h" diff --git a/src/common/compat.c b/src/common/compat.c index 7d9add50b2..6fdd6ecf00 100644 --- a/src/common/compat.c +++ b/src/common/compat.c @@ -100,7 +100,6 @@ SecureZeroMemory(PVOID ptr, SIZE_T cnt) /* Only use the linux prctl; the IRIX prctl is totally different */ #include <sys/prctl.h> #elif defined(__APPLE__) -#include <sys/types.h> #include <sys/ptrace.h> #endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) || ... */ @@ -116,7 +115,7 @@ SecureZeroMemory(PVOID ptr, SIZE_T cnt) #ifdef HAVE_SIGNAL_H #include <signal.h> #endif -#ifdef HAVE_SYS_MMAN_H +#ifdef HAVE_MMAP #include <sys/mman.h> #endif #ifdef HAVE_SYS_SYSLIMITS_H @@ -204,25 +203,17 @@ tor_rename(const char *path_old, const char *path_new) sandbox_intern_string(path_new)); } -/* Some MinGW builds have sys/mman.h, but not the corresponding symbols. - * Other configs rename the symbols using macros (including getpagesize). - * So check for sys/mman.h and unistd.h, and a getpagesize declaration. */ -#if (defined(HAVE_SYS_MMAN_H) && defined(HAVE_UNISTD_H) && \ - defined(HAVE_DECL_GETPAGESIZE)) -#define COMPAT_HAS_MMAN_AND_PAGESIZE -#endif - -#if defined(COMPAT_HAS_MMAN_AND_PAGESIZE) || \ - defined(RUNNING_DOXYGEN) +#if defined(HAVE_MMAP) || defined(RUNNING_DOXYGEN) /** Try to create a memory mapping for <b>filename</b> and return it. On - * failure, return NULL. Sets errno properly, using ERANGE to mean - * "empty file". */ + * failure, return NULL. Sets errno properly, using ERANGE to mean + * "empty file". Must only be called on trusted Tor-owned files, as changing + * the underlying file's size causes unspecified behavior. */ tor_mmap_t * tor_mmap_file(const char *filename) { int fd; /* router file */ char *string; - int page_size, result; + int result; tor_mmap_t *res; size_t size, filesize; struct stat st; @@ -251,13 +242,6 @@ tor_mmap_file(const char *filename) return NULL; } size = filesize = (size_t)(st.st_size); - /* - * Should we check for weird crap like mmapping a named pipe here, - * or just wait for if (!size) below to fail? - */ - /* ensure page alignment */ - page_size = getpagesize(); - size += (size%page_size) ? page_size-(size%page_size) : 0; if (st.st_size > SSIZE_T_CEILING || (off_t)size < st.st_size) { log_warn(LD_FS, "File \"%s\" is too large. Ignoring.",filename); @@ -418,40 +402,8 @@ tor_munmap_file(tor_mmap_t *handle) return 0; } #else -tor_mmap_t * -tor_mmap_file(const char *filename) -{ - struct stat st; - char *res = read_file_to_str(filename, RFTS_BIN|RFTS_IGNORE_MISSING, &st); - tor_mmap_t *handle; - if (! res) - return NULL; - handle = tor_malloc_zero(sizeof(tor_mmap_t)); - handle->data = res; - handle->size = st.st_size; - return handle; -} - -/** Unmap the file mapped with tor_mmap_file(), and return 0 for success - * or -1 for failure. - */ - -int -tor_munmap_file(tor_mmap_t *handle) -{ - char *d = NULL; - if (handle == NULL) - return 0; - - d = (char*)handle->data; - tor_free(d); - memwipe(handle, 0, sizeof(tor_mmap_t)); - tor_free(handle); - - /* Can't fail in this mmap()/munmap()-free case */ - return 0; -} -#endif /* defined(COMPAT_HAS_MMAN_AND_PAGESIZE) || ... || ... */ +#error "cannot implement tor_mmap_file" +#endif /* defined(HAVE_MMAP) || ... || ... */ /** Replacement for snprintf. Differs from platform snprintf in two * ways: First, always NUL-terminates its output. Second, always @@ -1392,6 +1344,24 @@ tor_getsockname,(tor_socket_t sock, struct sockaddr *address, return getsockname(sock, address, address_len); } +/** + * Find the local address associated with the socket <b>sock</b>, and + * place it in *<b>addr_out</b>. Return 0 on success, -1 on failure. + * + * (As tor_getsockname, but instead places the result in a tor_addr_t.) */ +int +tor_addr_from_getsockname(tor_addr_t *addr_out, tor_socket_t sock) +{ + struct sockaddr_storage ss; + socklen_t ss_len = sizeof(ss); + memset(&ss, 0, sizeof(ss)); + + if (tor_getsockname(sock, (struct sockaddr *) &ss, &ss_len) < 0) + return -1; + + return tor_addr_from_sockaddr(addr_out, (struct sockaddr *)&ss, NULL); +} + /** Turn <b>socket</b> into a nonblocking socket. Return 0 on success, -1 * on failure. */ diff --git a/src/common/compat.h b/src/common/compat.h index 3088e68355..c7e7f8d9ef 100644 --- a/src/common/compat.h +++ b/src/common/compat.h @@ -318,12 +318,12 @@ typedef struct tor_mmap_t { size_t size; /**< Size of the file. */ /* None of the fields below should be accessed from outside compat.c */ -#ifdef HAVE_SYS_MMAN_H +#ifdef HAVE_MMAP size_t mapping_size; /**< Size of the actual mapping. (This is this file * size, rounded up to the nearest page.) */ #elif defined _WIN32 HANDLE mmap_handle; -#endif /* defined(HAVE_SYS_MMAN_H) || ... */ +#endif /* defined(HAVE_MMAP) || ... */ } tor_mmap_t; @@ -510,6 +510,8 @@ int get_n_open_sockets(void); MOCK_DECL(int, tor_getsockname,(tor_socket_t socket, struct sockaddr *address, socklen_t *address_len)); +struct tor_addr_t; +int tor_addr_from_getsockname(struct tor_addr_t *addr_out, tor_socket_t sock); #define tor_socket_send(s, buf, len, flags) send(s, buf, len, flags) #define tor_socket_recv(s, buf, len, flags) recv(s, buf, len, flags) diff --git a/src/common/compat_libevent.c b/src/common/compat_libevent.c index 735385557c..e60eb148d8 100644 --- a/src/common/compat_libevent.c +++ b/src/common/compat_libevent.c @@ -11,7 +11,7 @@ #define COMPAT_LIBEVENT_PRIVATE #include "compat_libevent.h" -#include "crypto.h" +#include "crypto_rand.h" #include "util.h" #include "torlog.h" @@ -79,6 +79,43 @@ tor_event_free_(struct event *ev) /** Global event base for use by the main thread. */ static struct event_base *the_event_base = NULL; +/** + * @defgroup postloop post-loop event helpers + * + * If we're not careful, Libevent can susceptible to infinite event chains: + * one event can activate another, whose callback activates another, whose + * callback activates another, ad infinitum. While this is happening, + * Libevent won't be checking timeouts, socket-based events, signals, and so + * on. + * + * We solve this problem by marking some events as "post-loop". A post-loop + * event behaves like any ordinary event, but any events that _it_ activates + * cannot run until Libevent has checked for other events at least once. + * + * @{ */ + +/** + * An event that stops Libevent from running any more events on the current + * iteration of its loop, until it has re-checked for socket events, signal + * events, timeouts, etc. + */ +static struct event *rescan_mainloop_ev = NULL; + +/** + * Callback to implement rescan_mainloop_ev: it simply exits the mainloop, + * and relies on Tor to re-enter the mainloop since no error has occurred. + */ +static void +rescan_mainloop_cb(evutil_socket_t fd, short events, void *arg) +{ + (void)fd; + (void)events; + struct event_base *the_base = arg; + event_base_loopbreak(the_base); +} + +/** @} */ + /* This is what passes for version detection on OSX. We set * MACOSX_KQUEUE_IS_BROKEN to true iff we're on a version of OSX before * 10.4.0 (aka 1040). */ @@ -130,6 +167,15 @@ tor_libevent_initialize(tor_libevent_cfg *torcfg) /* LCOV_EXCL_STOP */ } + rescan_mainloop_ev = event_new(the_event_base, -1, 0, + rescan_mainloop_cb, the_event_base); + if (!rescan_mainloop_ev) { + /* LCOV_EXCL_START */ + log_err(LD_GENERAL, "Unable to create rescan event: cannot continue."); + exit(1); // exit ok: libevent is broken. + /* LCOV_EXCL_STOP */ + } + log_info(LD_GENERAL, "Initialized libevent version %s using method %s. Good.", event_get_version(), tor_libevent_get_method()); @@ -207,10 +253,39 @@ periodic_timer_new(struct event_base *base, } timer->cb = cb; timer->data = data; - event_add(timer->ev, (struct timeval *)tv); /*drop const for old libevent*/ + periodic_timer_launch(timer, tv); return timer; } +/** + * Launch the timer <b>timer</b> to run at <b>tv</b> from now, and every + * <b>tv</b> thereafter. + * + * If the timer is already enabled, this function does nothing. + */ +void +periodic_timer_launch(periodic_timer_t *timer, const struct timeval *tv) +{ + tor_assert(timer); + if (event_pending(timer->ev, EV_TIMEOUT, NULL)) + return; + event_add(timer->ev, tv); +} + +/** + * Disable the provided <b>timer</b>, but do not free it. + * + * You can reenable the same timer later with periodic_timer_launch. + * + * If the timer is already disabled, this function does nothing. + */ +void +periodic_timer_disable(periodic_timer_t *timer) +{ + tor_assert(timer); + (void) event_del(timer->ev); +} + /** Stop and free a periodic timer */ void periodic_timer_free_(periodic_timer_t *timer) @@ -221,6 +296,173 @@ periodic_timer_free_(periodic_timer_t *timer) tor_free(timer); } +/** + * Type used to represent events that run directly from the main loop, + * either because they are activated from elsewhere in the code, or + * because they have a simple timeout. + * + * We use this type to avoid exposing Libevent's API throughout the rest + * of the codebase. + * + * This type can't be used for all events: it doesn't handle events that + * are triggered by signals or by sockets. + */ +struct mainloop_event_t { + struct event *ev; + void (*cb)(mainloop_event_t *, void *); + void *userdata; +}; + +/** + * Internal: Implements mainloop event using a libevent event. + */ +static void +mainloop_event_cb(evutil_socket_t fd, short what, void *arg) +{ + (void)fd; + (void)what; + mainloop_event_t *mev = arg; + mev->cb(mev, mev->userdata); +} + +/** + * As mainloop_event_cb, but implements a post-loop event. + */ +static void +mainloop_event_postloop_cb(evutil_socket_t fd, short what, void *arg) +{ + (void)fd; + (void)what; + + /* Note that if rescan_mainloop_ev is already activated, + * event_active() will do nothing: only the first post-loop event that + * happens each time through the event loop will cause it to be + * activated. + * + * Because event_active() puts events on a FIFO queue, every event + * that is made active _after_ rescan_mainloop_ev will get its + * callback run after rescan_mainloop_cb is called -- that is, on the + * next iteration of the loop. + */ + event_active(rescan_mainloop_ev, EV_READ, 1); + + mainloop_event_t *mev = arg; + mev->cb(mev, mev->userdata); +} + +/** + * Helper for mainloop_event_new() and mainloop_event_postloop_new(). + */ +static mainloop_event_t * +mainloop_event_new_impl(int postloop, + void (*cb)(mainloop_event_t *, void *), + void *userdata) +{ + tor_assert(cb); + + struct event_base *base = tor_libevent_get_base(); + mainloop_event_t *mev = tor_malloc_zero(sizeof(mainloop_event_t)); + mev->ev = tor_event_new(base, -1, 0, + postloop ? mainloop_event_postloop_cb : mainloop_event_cb, + mev); + tor_assert(mev->ev); + mev->cb = cb; + mev->userdata = userdata; + return mev; +} + +/** + * Create and return a new mainloop_event_t to run the function <b>cb</b>. + * + * When run, the callback function will be passed the mainloop_event_t + * and <b>userdata</b> as its arguments. The <b>userdata</b> pointer + * must remain valid for as long as the mainloop_event_t event exists: + * it is your responsibility to free it. + * + * The event is not scheduled by default: Use mainloop_event_activate() + * or mainloop_event_schedule() to make it run. + */ +mainloop_event_t * +mainloop_event_new(void (*cb)(mainloop_event_t *, void *), + void *userdata) +{ + return mainloop_event_new_impl(0, cb, userdata); +} + +/** + * As mainloop_event_new(), but create a post-loop event. + * + * A post-loop event behaves like any ordinary event, but any events + * that _it_ activates cannot run until Libevent has checked for other + * events at least once. + */ +mainloop_event_t * +mainloop_event_postloop_new(void (*cb)(mainloop_event_t *, void *), + void *userdata) +{ + return mainloop_event_new_impl(1, cb, userdata); +} + +/** + * Schedule <b>event</b> to run in the main loop, immediately. If it is + * not scheduled, it will run anyway. If it is already scheduled to run + * later, it will run now instead. This function will have no effect if + * the event is already scheduled to run. + * + * This function may only be called from the main thread. + */ +void +mainloop_event_activate(mainloop_event_t *event) +{ + tor_assert(event); + event_active(event->ev, EV_READ, 1); +} + +/** Schedule <b>event</b> to run in the main loop, after a delay of <b>tv</b>. + * + * If the event is scheduled for a different time, cancel it and run + * after this delay instead. If the event is currently pending to run + * <em>now</b>, has no effect. + * + * Do not call this function with <b>tv</b> == NULL -- use + * mainloop_event_activate() instead. + * + * This function may only be called from the main thread. + */ +int +mainloop_event_schedule(mainloop_event_t *event, const struct timeval *tv) +{ + tor_assert(event); + if (BUG(tv == NULL)) { + // LCOV_EXCL_START + mainloop_event_activate(event); + return 0; + // LCOV_EXCL_STOP + } + return event_add(event->ev, tv); +} + +/** Cancel <b>event</b> if it is currently active or pending. (Do nothing if + * the event is not currently active or pending.) */ +void +mainloop_event_cancel(mainloop_event_t *event) +{ + if (!event) + return; + (void) event_del(event->ev); +} + +/** Cancel <b>event</b> and release all storage associated with it. */ +void +mainloop_event_free_(mainloop_event_t *event) +{ + if (!event) + return; + tor_event_free(event->ev); + memset(event, 0xb8, sizeof(*event)); + tor_free(event); +} + int tor_init_libevent_rng(void) { @@ -243,56 +485,45 @@ tor_init_libevent_rng(void) void tor_libevent_free_all(void) { + tor_event_free(rescan_mainloop_ev); if (the_event_base) event_base_free(the_event_base); the_event_base = NULL; } -#if defined(LIBEVENT_VERSION_NUMBER) && \ - LIBEVENT_VERSION_NUMBER >= V(2,1,1) && \ - !defined(TOR_UNIT_TESTS) -void -tor_gettimeofday_cached(struct timeval *tv) -{ - event_base_gettimeofday_cached(the_event_base, tv); -} -void -tor_gettimeofday_cache_clear(void) -{ - event_base_update_cache_time(the_event_base); -} -#else /* !(defined(LIBEVENT_VERSION_NUMBER) && ...) */ -/** Cache the current hi-res time; the cache gets reset when libevent - * calls us. */ -static struct timeval cached_time_hires = {0, 0}; - -/** Return a fairly recent view of the current time. */ -void -tor_gettimeofday_cached(struct timeval *tv) +/** + * Run the event loop for the provided event_base, handling events until + * something stops it. If <b>once</b> is set, then just poll-and-run + * once, then exit. Return 0 on success, -1 if an error occurred, or 1 + * if we exited because no events were pending or active. + * + * This isn't reentrant or multithreaded. + */ +int +tor_libevent_run_event_loop(struct event_base *base, int once) { - if (cached_time_hires.tv_sec == 0) { - tor_gettimeofday(&cached_time_hires); - } - *tv = cached_time_hires; + const int flags = once ? EVLOOP_ONCE : 0; + return event_base_loop(base, flags); } -/** Reset the cached view of the current time, so that the next time we try - * to learn it, we will get an up-to-date value. */ +/** Tell the event loop to exit after <b>delay</b>. If <b>delay</b> is NULL, + * instead exit after we're done running the currently active events. */ void -tor_gettimeofday_cache_clear(void) +tor_libevent_exit_loop_after_delay(struct event_base *base, + const struct timeval *delay) { - cached_time_hires.tv_sec = 0; + event_base_loopexit(base, delay); } -#ifdef TOR_UNIT_TESTS -/** For testing: force-update the cached time to a given value. */ +/** Tell the event loop to exit after running whichever callback is currently + * active. */ void -tor_gettimeofday_cache_set(const struct timeval *tv) +tor_libevent_exit_loop_after_callback(struct event_base *base) { - tor_assert(tv); - memcpy(&cached_time_hires, tv, sizeof(*tv)); + event_base_loopbreak(base); } +#if defined(TOR_UNIT_TESTS) /** For testing: called post-fork to make libevent reinitialize * kernel structures. */ void @@ -302,5 +533,4 @@ tor_libevent_postfork(void) tor_assert(r == 0); } #endif /* defined(TOR_UNIT_TESTS) */ -#endif /* defined(LIBEVENT_VERSION_NUMBER) && ... */ diff --git a/src/common/compat_libevent.h b/src/common/compat_libevent.h index 1853e50917..286a268122 100644 --- a/src/common/compat_libevent.h +++ b/src/common/compat_libevent.h @@ -7,8 +7,6 @@ #include "orconfig.h" #include "testsupport.h" -#include <event2/event.h> - void configure_libevent_logging(void); void suppress_libevent_log_msg(const char *msg); @@ -19,6 +17,9 @@ void suppress_libevent_log_msg(const char *msg); evdns_add_server_port_with_base(tor_libevent_get_base(), \ (sock),(tcp),(cb),(data)); +struct event; +struct event_base; + void tor_event_free_(struct event *ev); #define tor_event_free(ev) \ FREE_AND_NULL(struct event, tor_event_free_, (ev)) @@ -30,11 +31,24 @@ periodic_timer_t *periodic_timer_new(struct event_base *base, void (*cb)(periodic_timer_t *timer, void *data), void *data); void periodic_timer_free_(periodic_timer_t *); +void periodic_timer_launch(periodic_timer_t *, const struct timeval *tv); +void periodic_timer_disable(periodic_timer_t *); #define periodic_timer_free(t) \ FREE_AND_NULL(periodic_timer_t, periodic_timer_free_, (t)) -#define tor_event_base_loopexit event_base_loopexit -#define tor_event_base_loopbreak event_base_loopbreak +typedef struct mainloop_event_t mainloop_event_t; +mainloop_event_t *mainloop_event_new(void (*cb)(mainloop_event_t *, void *), + void *userdata); +mainloop_event_t * mainloop_event_postloop_new( + void (*cb)(mainloop_event_t *, void *), + void *userdata); +void mainloop_event_activate(mainloop_event_t *event); +int mainloop_event_schedule(mainloop_event_t *event, + const struct timeval *delay); +void mainloop_event_cancel(mainloop_event_t *event); +void mainloop_event_free_(mainloop_event_t *event); +#define mainloop_event_free(event) \ + FREE_AND_NULL(mainloop_event_t, mainloop_event_free_, (event)) /** Defines a configuration for using libevent with Tor: passed as an argument * to tor_libevent_initialize() to describe how we want to set up. */ @@ -56,13 +70,15 @@ void tor_libevent_free_all(void); int tor_init_libevent_rng(void); -void tor_gettimeofday_cached(struct timeval *tv); -void tor_gettimeofday_cache_clear(void); #ifdef TOR_UNIT_TESTS -void tor_gettimeofday_cache_set(const struct timeval *tv); void tor_libevent_postfork(void); #endif +int tor_libevent_run_event_loop(struct event_base *base, int once); +void tor_libevent_exit_loop_after_delay(struct event_base *base, + const struct timeval *delay); +void tor_libevent_exit_loop_after_callback(struct event_base *base); + #ifdef COMPAT_LIBEVENT_PRIVATE /** Macro: returns the number of a Libevent version as a 4-byte number, diff --git a/src/common/compat_time.c b/src/common/compat_time.c index 183a60a480..93b527def0 100644 --- a/src/common/compat_time.c +++ b/src/common/compat_time.c @@ -71,8 +71,8 @@ tor_sleep_msec(int msec) /** Set *timeval to the current time of day. On error, log and terminate. * (Same as gettimeofday(timeval,NULL), but never returns -1.) */ -void -tor_gettimeofday(struct timeval *timeval) +MOCK_IMPL(void, +tor_gettimeofday, (struct timeval *timeval)) { #ifdef _WIN32 /* Epoch bias copied from perl: number of units between windows epoch and @@ -279,6 +279,8 @@ monotime_reset_ratchets_for_testing(void) * nanoseconds. */ static struct mach_timebase_info mach_time_info; +static struct mach_timebase_info mach_time_info_msec_cvt; +static int32_t mach_time_msec_cvt_threshold; static int monotime_shift = 0; static void @@ -296,6 +298,18 @@ monotime_init_internal(void) // requires that tor_log2(0) == 0. monotime_shift = tor_log2(ms_per_tick); } + { + // For converting ticks to milliseconds in a 32-bit-friendly way, we + // will first right-shift by 20, and then multiply by 2048/1953, since + // (1<<20) * 1953/2048 is about 1e6. We precompute a new numerator and + // denominator here to avoid multiple multiplies. + mach_time_info_msec_cvt.numer = mach_time_info.numer * 2048; + mach_time_info_msec_cvt.denom = mach_time_info.denom * 1953; + // For any value above this amount, we should divide before multiplying, + // to avoid overflow. For a value below this, we should multiply + // before dividing, to improve accuracy. + mach_time_msec_cvt_threshold = INT32_MAX / mach_time_info_msec_cvt.numer; + } } /** @@ -345,6 +359,27 @@ monotime_diff_nsec(const monotime_t *start, return diff_nsec; } +int32_t +monotime_coarse_diff_msec32_(const monotime_coarse_t *start, + const monotime_coarse_t *end) +{ + if (BUG(mach_time_info.denom == 0)) { + monotime_init(); + } + const int64_t diff_ticks = end->abstime_ - start->abstime_; + + /* We already require in di_ops.c that right-shift performs a sign-extend. */ + const int32_t diff_microticks = (int32_t)(diff_ticks >> 20); + + if (diff_microticks >= mach_time_msec_cvt_threshold) { + return (diff_microticks / mach_time_info_msec_cvt.denom) * + mach_time_info_msec_cvt.numer; + } else { + return (diff_microticks * mach_time_info_msec_cvt.numer) / + mach_time_info_msec_cvt.denom; + } +} + uint32_t monotime_coarse_to_stamp(const monotime_coarse_t *t) { @@ -443,6 +478,15 @@ monotime_diff_nsec(const monotime_t *start, return diff_nsec; } +int32_t +monotime_coarse_diff_msec32_(const monotime_coarse_t *start, + const monotime_coarse_t *end) +{ + const int32_t diff_sec = (int32_t)(end->ts_.tv_sec - start->ts_.tv_sec); + const int32_t diff_nsec = (int32_t)(end->ts_.tv_nsec - start->ts_.tv_nsec); + return diff_sec * 1000 + diff_nsec / ONE_MILLION; +} + /* This value is ONE_BILLION >> 20. */ static const uint32_t STAMP_TICKS_PER_SECOND = 953; @@ -592,6 +636,13 @@ monotime_coarse_diff_msec(const monotime_coarse_t *start, return diff_ticks; } +int32_t +monotime_coarse_diff_msec32_(const monotime_coarse_t *start, + const monotime_coarse_t *end) +{ + return (int32_t)monotime_coarse_diff_msec(start, end); +} + int64_t monotime_coarse_diff_usec(const monotime_coarse_t *start, const monotime_coarse_t *end) @@ -677,6 +728,15 @@ monotime_diff_nsec(const monotime_t *start, return (diff.tv_sec * ONE_BILLION + diff.tv_usec * 1000); } +int32_t +monotime_coarse_diff_msec32_(const monotime_coarse_t *start, + const monotime_coarse_t *end) +{ + struct timeval diff; + timersub(&end->tv_, &start->tv_, &diff); + return diff.tv_sec * 1000 + diff.tv_usec / 1000; +} + /* This value is ONE_MILLION >> 10. */ static const uint32_t STAMP_TICKS_PER_SECOND = 976; @@ -830,11 +890,24 @@ monotime_coarse_stamp_units_to_approx_msec(uint64_t units) return (abstime_diff * mach_time_info.numer) / (mach_time_info.denom * ONE_MILLION); } +uint64_t +monotime_msec_to_approx_coarse_stamp_units(uint64_t msec) +{ + uint64_t abstime_val = + (((uint64_t)msec) * ONE_MILLION * mach_time_info.denom) / + mach_time_info.numer; + return abstime_val >> monotime_shift; +} #else uint64_t monotime_coarse_stamp_units_to_approx_msec(uint64_t units) { return (units * 1000) / STAMP_TICKS_PER_SECOND; } +uint64_t +monotime_msec_to_approx_coarse_stamp_units(uint64_t msec) +{ + return (msec * STAMP_TICKS_PER_SECOND) / 1000; +} #endif diff --git a/src/common/compat_time.h b/src/common/compat_time.h index 6ddd11883d..f241aa5eba 100644 --- a/src/common/compat_time.h +++ b/src/common/compat_time.h @@ -150,6 +150,7 @@ uint32_t monotime_coarse_to_stamp(const monotime_coarse_t *t); * into an approximate number of milliseconds. */ uint64_t monotime_coarse_stamp_units_to_approx_msec(uint64_t units); +uint64_t monotime_msec_to_approx_coarse_stamp_units(uint64_t msec); uint32_t monotime_coarse_get_stamp(void); #if defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT) @@ -172,7 +173,35 @@ void monotime_coarse_add_msec(monotime_coarse_t *out, #define monotime_coarse_add_msec monotime_add_msec #endif /* defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT) */ -void tor_gettimeofday(struct timeval *timeval); +/** + * As monotime_coarse_diff_msec, but avoid 64-bit division. + * + * Requires that the difference fit into an int32_t; not for use with + * large time differences. + */ +int32_t monotime_coarse_diff_msec32_(const monotime_coarse_t *start, + const monotime_coarse_t *end); + +/** + * As monotime_coarse_diff_msec, but avoid 64-bit division if it is expensive. + * + * Requires that the difference fit into an int32_t; not for use with + * large time differences. + */ +static inline int32_t +monotime_coarse_diff_msec32(const monotime_coarse_t *start, + const monotime_coarse_t *end) +{ +#if SIZEOF_VOID_P == 8 + // on a 64-bit platform, let's assume 64/64 division is cheap. + return (int32_t) monotime_coarse_diff_msec(start, end); +#else +#define USING_32BIT_MSEC_HACK + return monotime_coarse_diff_msec32_(start, end); +#endif +} + +MOCK_DECL(void, tor_gettimeofday, (struct timeval *timeval)); #ifdef TOR_UNIT_TESTS void tor_sleep_msec(int msec); diff --git a/src/common/compat_winthreads.c b/src/common/compat_winthreads.c index 5f7ec94c23..7021344f6e 100644 --- a/src/common/compat_winthreads.c +++ b/src/common/compat_winthreads.c @@ -18,7 +18,6 @@ #include "util.h" #include "container.h" #include "torlog.h" -#include <process.h> /* This value is more or less total cargo-cult */ #define SPIN_COUNT 2000 diff --git a/src/common/compress.c b/src/common/compress.c index 47c93cf6a9..cb1549f1aa 100644 --- a/src/common/compress.c +++ b/src/common/compress.c @@ -663,3 +663,13 @@ tor_compress_init(void) tor_zstd_init(); } +/** Warn if we had any problems while setting up our compression libraries. + * + * (This isn't part of tor_compress_init, since the logs aren't set up yet.) + */ +void +tor_compress_log_init_warnings(void) +{ + tor_zstd_warn_if_version_mismatched(); +} + diff --git a/src/common/compress.h b/src/common/compress.h index 952102bf97..65d63a4386 100644 --- a/src/common/compress.h +++ b/src/common/compress.h @@ -87,6 +87,7 @@ void tor_compress_free_(tor_compress_state_t *state); size_t tor_compress_state_size(const tor_compress_state_t *state); void tor_compress_init(void); +void tor_compress_log_init_warnings(void); #endif /* !defined(TOR_COMPRESS_H) */ diff --git a/src/common/compress_zstd.c b/src/common/compress_zstd.c index b9f9f1f076..dc8b4d621d 100644 --- a/src/common/compress_zstd.c +++ b/src/common/compress_zstd.c @@ -18,6 +18,13 @@ #include "compress.h" #include "compress_zstd.h" +#ifdef ENABLE_ZSTD_ADVANCED_APIS +/* This is a lie, but we make sure it doesn't get us in trouble by wrapping + * all invocations of zstd's static-only functions in a check to make sure + * that the compile-time version matches the run-time version. */ +#define ZSTD_STATIC_LINKING_ONLY +#endif + #ifdef HAVE_ZSTD #ifdef HAVE_CFLAG_WUNUSED_CONST_VARIABLE DISABLE_GCC_WARNING(unused-const-variable) @@ -57,21 +64,31 @@ tor_zstd_method_supported(void) #endif } +#ifdef HAVE_ZSTD +/** Format a zstd version number as a string in <b>buf</b>. */ +static void +tor_zstd_format_version(char *buf, size_t buflen, unsigned version_number) +{ + tor_snprintf(buf, buflen, + "%u.%u.%u", + version_number / 10000 % 100, + version_number / 100 % 100, + version_number % 100); +} +#endif + +#define VERSION_STR_MAX_LEN 16 /* more than enough space for 99.99.99 */ + /** Return a string representation of the version of the currently running * version of libzstd. Returns NULL if Zstandard is unsupported. */ const char * tor_zstd_get_version_str(void) { #ifdef HAVE_ZSTD - static char version_str[16]; - size_t version_number; + static char version_str[VERSION_STR_MAX_LEN]; - version_number = ZSTD_versionNumber(); - tor_snprintf(version_str, sizeof(version_str), - "%d.%d.%d", - (int) version_number / 10000 % 100, - (int) version_number / 100 % 100, - (int) version_number % 100); + tor_zstd_format_version(version_str, sizeof(version_str), + ZSTD_versionNumber()); return version_str; #else /* !(defined(HAVE_ZSTD)) */ @@ -91,6 +108,26 @@ tor_zstd_get_header_version_str(void) #endif } +#ifdef TOR_UNIT_TESTS +static int static_apis_disable_for_testing = 0; +#endif + +/** Return true iff we can use the "static-only" APIs. */ +int +tor_zstd_can_use_static_apis(void) +{ +#if defined(ZSTD_STATIC_LINKING_ONLY) && defined(HAVE_ZSTD) +#ifdef TOR_UNIT_TESTS + if (static_apis_disable_for_testing) { + return 0; + } +#endif + return (ZSTD_VERSION_NUMBER == ZSTD_versionNumber()); +#else + return 0; +#endif +} + /** Internal Zstandard state for incremental compression/decompression. * The body of this struct is not exposed. */ struct tor_zstd_compress_state_t { @@ -118,9 +155,11 @@ struct tor_zstd_compress_state_t { #ifdef HAVE_ZSTD /** Return an approximate number of bytes stored in memory to hold the - * Zstandard compression/decompression state. */ + * Zstandard compression/decompression state. This is a fake estimate + * based on inspecting the zstd source: tor_zstd_state_size_precalc() is + * more accurate when it's allowed to use "static-only" functions */ static size_t -tor_zstd_state_size_precalc(int compress, int preset) +tor_zstd_state_size_precalc_fake(int compress, int preset) { tor_assert(preset > 0); @@ -177,6 +216,28 @@ tor_zstd_state_size_precalc(int compress, int preset) return memory_usage; } + +/** Return an approximate number of bytes stored in memory to hold the + * Zstandard compression/decompression state. */ +static size_t +tor_zstd_state_size_precalc(int compress, int preset) +{ +#ifdef ZSTD_STATIC_LINKING_ONLY + if (tor_zstd_can_use_static_apis()) { + if (compress) { +#ifdef HAVE_ZSTD_ESTIMATECSTREAMSIZE + return ZSTD_estimateCStreamSize(preset); +#endif + } else { +#ifdef HAVE_ZSTD_ESTIMATEDCTXSIZE + /* Could use DStream, but that takes a windowSize. */ + return ZSTD_estimateDCtxSize(); +#endif + } + } +#endif + return tor_zstd_state_size_precalc_fake(compress, preset); +} #endif /* defined(HAVE_ZSTD) */ /** Construct and return a tor_zstd_compress_state_t object using @@ -446,3 +507,34 @@ tor_zstd_init(void) atomic_counter_init(&total_zstd_allocation); } +/** Warn if the header and library versions don't match. */ +void +tor_zstd_warn_if_version_mismatched(void) +{ +#if defined(HAVE_ZSTD) && defined(ENABLE_ZSTD_ADVANCED_APIS) + if (! tor_zstd_can_use_static_apis()) { + char header_version[VERSION_STR_MAX_LEN]; + char runtime_version[VERSION_STR_MAX_LEN]; + tor_zstd_format_version(header_version, sizeof(header_version), + ZSTD_VERSION_NUMBER); + tor_zstd_format_version(runtime_version, sizeof(runtime_version), + ZSTD_versionNumber()); + + log_warn(LD_GENERAL, + "Tor was compiled with zstd %s, but is running with zstd %s. " + "For safety, we'll avoid using advanced zstd functionality.", + header_version, runtime_version); + } +#endif +} + +#ifdef TOR_UNIT_TESTS +/** Testing only: disable usage of static-only APIs, so we can make sure that + * we still work without them. */ +void +tor_zstd_set_static_apis_disabled_for_testing(int disabled) +{ + static_apis_disable_for_testing = disabled; +} +#endif + diff --git a/src/common/compress_zstd.h b/src/common/compress_zstd.h index 9bca24ded7..bd42cf65ce 100644 --- a/src/common/compress_zstd.h +++ b/src/common/compress_zstd.h @@ -17,6 +17,8 @@ const char *tor_zstd_get_version_str(void); const char *tor_zstd_get_header_version_str(void); +int tor_zstd_can_use_static_apis(void); + /** Internal state for an incremental Zstandard compression/decompression. */ typedef struct tor_zstd_compress_state_t tor_zstd_compress_state_t; @@ -41,6 +43,11 @@ size_t tor_zstd_compress_state_size(const tor_zstd_compress_state_t *state); size_t tor_zstd_get_total_allocation(void); void tor_zstd_init(void); +void tor_zstd_warn_if_version_mismatched(void); + +#ifdef TOR_UNIT_TESTS +void tor_zstd_set_static_apis_disabled_for_testing(int disabled); +#endif #endif /* !defined(TOR_COMPRESS_ZSTD_H) */ diff --git a/src/common/container.c b/src/common/container.c index 54b0b2028f..5386e6458b 100644 --- a/src/common/container.c +++ b/src/common/container.c @@ -15,7 +15,7 @@ #include "util.h" #include "torlog.h" #include "container.h" -#include "crypto.h" +#include "crypto_digest.h" #include <stdlib.h> #include <string.h> diff --git a/src/common/crypto.c b/src/common/crypto.c index d85aca4004..d5b7c96916 100644 --- a/src/common/crypto.c +++ b/src/common/crypto.c @@ -23,25 +23,26 @@ #endif /* defined(_WIN32) */ #define CRYPTO_PRIVATE -#include "crypto.h" #include "compat_openssl.h" +#include "crypto.h" #include "crypto_curve25519.h" +#include "crypto_digest.h" #include "crypto_ed25519.h" #include "crypto_format.h" +#include "crypto_rand.h" #include "crypto_rsa.h" +#include "crypto_util.h" DISABLE_GCC_WARNING(redundant-decls) #include <openssl/err.h> -#include <openssl/rsa.h> -#include <openssl/pem.h> #include <openssl/evp.h> #include <openssl/engine.h> -#include <openssl/rand.h> #include <openssl/bn.h> #include <openssl/dh.h> #include <openssl/conf.h> #include <openssl/hmac.h> +#include <openssl/ssl.h> ENABLE_GCC_WARNING(redundant-decls) @@ -59,18 +60,6 @@ ENABLE_GCC_WARNING(redundant-decls) #ifdef HAVE_UNISTD_H #include <unistd.h> #endif -#ifdef HAVE_FCNTL_H -#include <fcntl.h> -#endif -#ifdef HAVE_SYS_FCNTL_H -#include <sys/fcntl.h> -#endif -#ifdef HAVE_SYS_SYSCALL_H -#include <sys/syscall.h> -#endif -#ifdef HAVE_SYS_RANDOM_H -#include <sys/random.h> -#endif #include "torlog.h" #include "torint.h" @@ -83,12 +72,6 @@ ENABLE_GCC_WARNING(redundant-decls) #include "keccak-tiny/keccak-tiny.h" -/** Longest recognized */ -#define MAX_DNS_LABEL_SIZE 63 - -/** Largest strong entropy request */ -#define MAX_STRONGEST_RAND_SIZE 256 - /** A structure to hold the first half (x, g^x) of a Diffie-Hellman handshake * while we're waiting for the second.*/ struct crypto_dh_t { @@ -161,23 +144,6 @@ try_load_engine(const char *path, const char *engine) } #endif /* !defined(DISABLE_ENGINES) */ -/** Make sure that openssl is using its default PRNG. Return 1 if we had to - * adjust it; 0 otherwise. */ -STATIC int -crypto_force_rand_ssleay(void) -{ - RAND_METHOD *default_method; - default_method = RAND_OpenSSL(); - if (RAND_get_rand_method() != default_method) { - log_notice(LD_CRYPTO, "It appears that one of our engines has provided " - "a replacement the OpenSSL RNG. Resetting it to the default " - "implementation."); - RAND_set_rand_method(default_method); - return 1; - } - return 0; -} - static int have_seeded_siphash = 0; /** Set up the siphash key if we haven't already done so. */ @@ -203,8 +169,15 @@ crypto_early_init(void) crypto_early_initialized_ = 1; +#ifdef OPENSSL_1_1_API + OPENSSL_init_ssl(OPENSSL_INIT_LOAD_SSL_STRINGS | + OPENSSL_INIT_LOAD_CRYPTO_STRINGS | + OPENSSL_INIT_ADD_ALL_CIPHERS | + OPENSSL_INIT_ADD_ALL_DIGESTS, NULL); +#else ERR_load_crypto_strings(); OpenSSL_add_all_algorithms(); +#endif setup_openssl_threading(); @@ -397,266 +370,6 @@ crypto_cipher_free_(crypto_cipher_t *env) aes_cipher_free(env); } -/* public key crypto */ - -/** Check a siglen-byte long signature at <b>sig</b> against - * <b>datalen</b> bytes of data at <b>data</b>, using the public key - * in <b>env</b>. Return 0 if <b>sig</b> is a correct signature for - * SHA1(data). Else return -1. - */ -MOCK_IMPL(int, -crypto_pk_public_checksig_digest,(crypto_pk_t *env, const char *data, - size_t datalen, const char *sig, - size_t siglen)) -{ - char digest[DIGEST_LEN]; - char *buf; - size_t buflen; - int r; - - tor_assert(env); - tor_assert(data); - tor_assert(sig); - tor_assert(datalen < SIZE_T_CEILING); - tor_assert(siglen < SIZE_T_CEILING); - - if (crypto_digest(digest,data,datalen)<0) { - log_warn(LD_BUG, "couldn't compute digest"); - return -1; - } - buflen = crypto_pk_keysize(env); - buf = tor_malloc(buflen); - r = crypto_pk_public_checksig(env,buf,buflen,sig,siglen); - if (r != DIGEST_LEN) { - log_warn(LD_CRYPTO, "Invalid signature"); - tor_free(buf); - return -1; - } - if (tor_memneq(buf, digest, DIGEST_LEN)) { - log_warn(LD_CRYPTO, "Signature mismatched with digest."); - tor_free(buf); - return -1; - } - tor_free(buf); - - return 0; -} - -/** Compute a SHA1 digest of <b>fromlen</b> bytes of data stored at - * <b>from</b>; sign the data with the private key in <b>env</b>, and - * store it in <b>to</b>. Return the number of bytes written on - * success, and -1 on failure. - * - * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be - * at least the length of the modulus of <b>env</b>. - */ -int -crypto_pk_private_sign_digest(crypto_pk_t *env, char *to, size_t tolen, - const char *from, size_t fromlen) -{ - int r; - char digest[DIGEST_LEN]; - if (crypto_digest(digest,from,fromlen)<0) - return -1; - r = crypto_pk_private_sign(env,to,tolen,digest,DIGEST_LEN); - memwipe(digest, 0, sizeof(digest)); - return r; -} - -/** Perform a hybrid (public/secret) encryption on <b>fromlen</b> - * bytes of data from <b>from</b>, with padding type 'padding', - * storing the results on <b>to</b>. - * - * Returns the number of bytes written on success, -1 on failure. - * - * The encrypted data consists of: - * - The source data, padded and encrypted with the public key, if the - * padded source data is no longer than the public key, and <b>force</b> - * is false, OR - * - The beginning of the source data prefixed with a 16-byte symmetric key, - * padded and encrypted with the public key; followed by the rest of - * the source data encrypted in AES-CTR mode with the symmetric key. - * - * NOTE that this format does not authenticate the symmetrically encrypted - * part of the data, and SHOULD NOT BE USED for new protocols. - */ -int -crypto_pk_obsolete_public_hybrid_encrypt(crypto_pk_t *env, - char *to, size_t tolen, - const char *from, - size_t fromlen, - int padding, int force) -{ - int overhead, outlen, r; - size_t pkeylen, symlen; - crypto_cipher_t *cipher = NULL; - char *buf = NULL; - - tor_assert(env); - tor_assert(from); - tor_assert(to); - tor_assert(fromlen < SIZE_T_CEILING); - - overhead = crypto_get_rsa_padding_overhead(crypto_get_rsa_padding(padding)); - pkeylen = crypto_pk_keysize(env); - - if (!force && fromlen+overhead <= pkeylen) { - /* It all fits in a single encrypt. */ - return crypto_pk_public_encrypt(env,to, - tolen, - from,fromlen,padding); - } - tor_assert(tolen >= fromlen + overhead + CIPHER_KEY_LEN); - tor_assert(tolen >= pkeylen); - - char key[CIPHER_KEY_LEN]; - crypto_rand(key, sizeof(key)); /* generate a new key. */ - cipher = crypto_cipher_new(key); - - buf = tor_malloc(pkeylen+1); - memcpy(buf, key, CIPHER_KEY_LEN); - memcpy(buf+CIPHER_KEY_LEN, from, pkeylen-overhead-CIPHER_KEY_LEN); - - /* Length of symmetrically encrypted data. */ - symlen = fromlen-(pkeylen-overhead-CIPHER_KEY_LEN); - - outlen = crypto_pk_public_encrypt(env,to,tolen,buf,pkeylen-overhead,padding); - if (outlen!=(int)pkeylen) { - goto err; - } - r = crypto_cipher_encrypt(cipher, to+outlen, - from+pkeylen-overhead-CIPHER_KEY_LEN, symlen); - - if (r<0) goto err; - memwipe(buf, 0, pkeylen); - memwipe(key, 0, sizeof(key)); - tor_free(buf); - crypto_cipher_free(cipher); - tor_assert(outlen+symlen < INT_MAX); - return (int)(outlen + symlen); - err: - - memwipe(buf, 0, pkeylen); - memwipe(key, 0, sizeof(key)); - tor_free(buf); - crypto_cipher_free(cipher); - return -1; -} - -/** Invert crypto_pk_obsolete_public_hybrid_encrypt. Returns the number of - * bytes written on success, -1 on failure. - * - * NOTE that this format does not authenticate the symmetrically encrypted - * part of the data, and SHOULD NOT BE USED for new protocols. - */ -int -crypto_pk_obsolete_private_hybrid_decrypt(crypto_pk_t *env, - char *to, - size_t tolen, - const char *from, - size_t fromlen, - int padding, int warnOnFailure) -{ - int outlen, r; - size_t pkeylen; - crypto_cipher_t *cipher = NULL; - char *buf = NULL; - - tor_assert(fromlen < SIZE_T_CEILING); - pkeylen = crypto_pk_keysize(env); - - if (fromlen <= pkeylen) { - return crypto_pk_private_decrypt(env,to,tolen,from,fromlen,padding, - warnOnFailure); - } - - buf = tor_malloc(pkeylen); - outlen = crypto_pk_private_decrypt(env,buf,pkeylen,from,pkeylen,padding, - warnOnFailure); - if (outlen<0) { - log_fn(warnOnFailure?LOG_WARN:LOG_DEBUG, LD_CRYPTO, - "Error decrypting public-key data"); - goto err; - } - if (outlen < CIPHER_KEY_LEN) { - log_fn(warnOnFailure?LOG_WARN:LOG_INFO, LD_CRYPTO, - "No room for a symmetric key"); - goto err; - } - cipher = crypto_cipher_new(buf); - if (!cipher) { - goto err; - } - memcpy(to,buf+CIPHER_KEY_LEN,outlen-CIPHER_KEY_LEN); - outlen -= CIPHER_KEY_LEN; - tor_assert(tolen - outlen >= fromlen - pkeylen); - r = crypto_cipher_decrypt(cipher, to+outlen, from+pkeylen, fromlen-pkeylen); - if (r<0) - goto err; - memwipe(buf,0,pkeylen); - tor_free(buf); - crypto_cipher_free(cipher); - tor_assert(outlen + fromlen < INT_MAX); - return (int)(outlen + (fromlen-pkeylen)); - err: - memwipe(buf,0,pkeylen); - tor_free(buf); - crypto_cipher_free(cipher); - return -1; -} - -/** Given a private or public key <b>pk</b>, put a SHA1 hash of the - * public key into <b>digest_out</b> (must have DIGEST_LEN bytes of space). - * Return 0 on success, -1 on failure. - */ -int -crypto_pk_get_digest(const crypto_pk_t *pk, char *digest_out) -{ - char *buf; - size_t buflen; - int len; - int rv = -1; - - buflen = crypto_pk_keysize(pk)*2; - buf = tor_malloc(buflen); - len = crypto_pk_asn1_encode(pk, buf, buflen); - if (len < 0) - goto done; - - if (crypto_digest(digest_out, buf, len) < 0) - goto done; - - rv = 0; - done: - tor_free(buf); - return rv; -} - -/** Compute all digests of the DER encoding of <b>pk</b>, and store them - * in <b>digests_out</b>. Return 0 on success, -1 on failure. */ -int -crypto_pk_get_common_digests(crypto_pk_t *pk, common_digests_t *digests_out) -{ - char *buf; - size_t buflen; - int len; - int rv = -1; - - buflen = crypto_pk_keysize(pk)*2; - buf = tor_malloc(buflen); - len = crypto_pk_asn1_encode(pk, buf, buflen); - if (len < 0) - goto done; - - if (crypto_common_digests(digests_out, (char*)buf, len) < 0) - goto done; - - rv = 0; - done: - tor_free(buf); - return rv; -} - /** Copy <b>in</b> to the <b>outlen</b>-byte buffer <b>out</b>, adding spaces * every four characters. */ void @@ -788,524 +501,6 @@ crypto_cipher_decrypt_with_iv(const char *key, return (int)(fromlen - CIPHER_IV_LEN); } -/* SHA-1 */ - -/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in - * <b>m</b>. Write the DIGEST_LEN byte result into <b>digest</b>. - * Return 0 on success, -1 on failure. - */ -int -crypto_digest(char *digest, const char *m, size_t len) -{ - tor_assert(m); - tor_assert(digest); - if (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL) - return -1; - return 0; -} - -/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>, - * using the algorithm <b>algorithm</b>. Write the DIGEST_LEN256-byte result - * into <b>digest</b>. Return 0 on success, -1 on failure. */ -int -crypto_digest256(char *digest, const char *m, size_t len, - digest_algorithm_t algorithm) -{ - tor_assert(m); - tor_assert(digest); - tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); - - int ret = 0; - if (algorithm == DIGEST_SHA256) - ret = (SHA256((const uint8_t*)m,len,(uint8_t*)digest) != NULL); - else - ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len) - > -1); - - if (!ret) - return -1; - return 0; -} - -/** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>, - * using the algorithm <b>algorithm</b>. Write the DIGEST_LEN512-byte result - * into <b>digest</b>. Return 0 on success, -1 on failure. */ -int -crypto_digest512(char *digest, const char *m, size_t len, - digest_algorithm_t algorithm) -{ - tor_assert(m); - tor_assert(digest); - tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); - - int ret = 0; - if (algorithm == DIGEST_SHA512) - ret = (SHA512((const unsigned char*)m,len,(unsigned char*)digest) - != NULL); - else - ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len) - > -1); - - if (!ret) - return -1; - return 0; -} - -/** Set the common_digests_t in <b>ds_out</b> to contain every digest on the - * <b>len</b> bytes in <b>m</b> that we know how to compute. Return 0 on - * success, -1 on failure. */ -int -crypto_common_digests(common_digests_t *ds_out, const char *m, size_t len) -{ - tor_assert(ds_out); - memset(ds_out, 0, sizeof(*ds_out)); - if (crypto_digest(ds_out->d[DIGEST_SHA1], m, len) < 0) - return -1; - if (crypto_digest256(ds_out->d[DIGEST_SHA256], m, len, DIGEST_SHA256) < 0) - return -1; - - return 0; -} - -/** Return the name of an algorithm, as used in directory documents. */ -const char * -crypto_digest_algorithm_get_name(digest_algorithm_t alg) -{ - switch (alg) { - case DIGEST_SHA1: - return "sha1"; - case DIGEST_SHA256: - return "sha256"; - case DIGEST_SHA512: - return "sha512"; - case DIGEST_SHA3_256: - return "sha3-256"; - case DIGEST_SHA3_512: - return "sha3-512"; - // LCOV_EXCL_START - default: - tor_fragile_assert(); - return "??unknown_digest??"; - // LCOV_EXCL_STOP - } -} - -/** Given the name of a digest algorithm, return its integer value, or -1 if - * the name is not recognized. */ -int -crypto_digest_algorithm_parse_name(const char *name) -{ - if (!strcmp(name, "sha1")) - return DIGEST_SHA1; - else if (!strcmp(name, "sha256")) - return DIGEST_SHA256; - else if (!strcmp(name, "sha512")) - return DIGEST_SHA512; - else if (!strcmp(name, "sha3-256")) - return DIGEST_SHA3_256; - else if (!strcmp(name, "sha3-512")) - return DIGEST_SHA3_512; - else - return -1; -} - -/** Given an algorithm, return the digest length in bytes. */ -size_t -crypto_digest_algorithm_get_length(digest_algorithm_t alg) -{ - switch (alg) { - case DIGEST_SHA1: - return DIGEST_LEN; - case DIGEST_SHA256: - return DIGEST256_LEN; - case DIGEST_SHA512: - return DIGEST512_LEN; - case DIGEST_SHA3_256: - return DIGEST256_LEN; - case DIGEST_SHA3_512: - return DIGEST512_LEN; - default: - tor_assert(0); // LCOV_EXCL_LINE - return 0; /* Unreachable */ // LCOV_EXCL_LINE - } -} - -/** Intermediate information about the digest of a stream of data. */ -struct crypto_digest_t { - digest_algorithm_t algorithm; /**< Which algorithm is in use? */ - /** State for the digest we're using. Only one member of the - * union is usable, depending on the value of <b>algorithm</b>. Note also - * that space for other members might not even be allocated! - */ - union { - SHA_CTX sha1; /**< state for SHA1 */ - SHA256_CTX sha2; /**< state for SHA256 */ - SHA512_CTX sha512; /**< state for SHA512 */ - keccak_state sha3; /**< state for SHA3-[256,512] */ - } d; -}; - -#ifdef TOR_UNIT_TESTS - -digest_algorithm_t -crypto_digest_get_algorithm(crypto_digest_t *digest) -{ - tor_assert(digest); - - return digest->algorithm; -} - -#endif /* defined(TOR_UNIT_TESTS) */ - -/** - * Return the number of bytes we need to malloc in order to get a - * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe - * when we free one. - */ -static size_t -crypto_digest_alloc_bytes(digest_algorithm_t alg) -{ - /* Helper: returns the number of bytes in the 'f' field of 'st' */ -#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f )) - /* Gives the length of crypto_digest_t through the end of the field 'd' */ -#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \ - STRUCT_FIELD_SIZE(crypto_digest_t, f)) - switch (alg) { - case DIGEST_SHA1: - return END_OF_FIELD(d.sha1); - case DIGEST_SHA256: - return END_OF_FIELD(d.sha2); - case DIGEST_SHA512: - return END_OF_FIELD(d.sha512); - case DIGEST_SHA3_256: - case DIGEST_SHA3_512: - return END_OF_FIELD(d.sha3); - default: - tor_assert(0); // LCOV_EXCL_LINE - return 0; // LCOV_EXCL_LINE - } -#undef END_OF_FIELD -#undef STRUCT_FIELD_SIZE -} - -/** - * Internal function: create and return a new digest object for 'algorithm'. - * Does not typecheck the algorithm. - */ -static crypto_digest_t * -crypto_digest_new_internal(digest_algorithm_t algorithm) -{ - crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm)); - r->algorithm = algorithm; - - switch (algorithm) - { - case DIGEST_SHA1: - SHA1_Init(&r->d.sha1); - break; - case DIGEST_SHA256: - SHA256_Init(&r->d.sha2); - break; - case DIGEST_SHA512: - SHA512_Init(&r->d.sha512); - break; - case DIGEST_SHA3_256: - keccak_digest_init(&r->d.sha3, 256); - break; - case DIGEST_SHA3_512: - keccak_digest_init(&r->d.sha3, 512); - break; - default: - tor_assert_unreached(); - } - - return r; -} - -/** Allocate and return a new digest object to compute SHA1 digests. - */ -crypto_digest_t * -crypto_digest_new(void) -{ - return crypto_digest_new_internal(DIGEST_SHA1); -} - -/** Allocate and return a new digest object to compute 256-bit digests - * using <b>algorithm</b>. */ -crypto_digest_t * -crypto_digest256_new(digest_algorithm_t algorithm) -{ - tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); - return crypto_digest_new_internal(algorithm); -} - -/** Allocate and return a new digest object to compute 512-bit digests - * using <b>algorithm</b>. */ -crypto_digest_t * -crypto_digest512_new(digest_algorithm_t algorithm) -{ - tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); - return crypto_digest_new_internal(algorithm); -} - -/** Deallocate a digest object. - */ -void -crypto_digest_free_(crypto_digest_t *digest) -{ - if (!digest) - return; - size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); - memwipe(digest, 0, bytes); - tor_free(digest); -} - -/** Add <b>len</b> bytes from <b>data</b> to the digest object. - */ -void -crypto_digest_add_bytes(crypto_digest_t *digest, const char *data, - size_t len) -{ - tor_assert(digest); - tor_assert(data); - /* Using the SHA*_*() calls directly means we don't support doing - * SHA in hardware. But so far the delay of getting the question - * to the hardware, and hearing the answer, is likely higher than - * just doing it ourselves. Hashes are fast. - */ - switch (digest->algorithm) { - case DIGEST_SHA1: - SHA1_Update(&digest->d.sha1, (void*)data, len); - break; - case DIGEST_SHA256: - SHA256_Update(&digest->d.sha2, (void*)data, len); - break; - case DIGEST_SHA512: - SHA512_Update(&digest->d.sha512, (void*)data, len); - break; - case DIGEST_SHA3_256: /* FALLSTHROUGH */ - case DIGEST_SHA3_512: - keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len); - break; - default: - /* LCOV_EXCL_START */ - tor_fragile_assert(); - break; - /* LCOV_EXCL_STOP */ - } -} - -/** Compute the hash of the data that has been passed to the digest - * object; write the first out_len bytes of the result to <b>out</b>. - * <b>out_len</b> must be \<= DIGEST512_LEN. - */ -void -crypto_digest_get_digest(crypto_digest_t *digest, - char *out, size_t out_len) -{ - unsigned char r[DIGEST512_LEN]; - crypto_digest_t tmpenv; - tor_assert(digest); - tor_assert(out); - tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm)); - - /* The SHA-3 code handles copying into a temporary ctx, and also can handle - * short output buffers by truncating appropriately. */ - if (digest->algorithm == DIGEST_SHA3_256 || - digest->algorithm == DIGEST_SHA3_512) { - keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len); - return; - } - - const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm); - /* memcpy into a temporary ctx, since SHA*_Final clears the context */ - memcpy(&tmpenv, digest, alloc_bytes); - switch (digest->algorithm) { - case DIGEST_SHA1: - SHA1_Final(r, &tmpenv.d.sha1); - break; - case DIGEST_SHA256: - SHA256_Final(r, &tmpenv.d.sha2); - break; - case DIGEST_SHA512: - SHA512_Final(r, &tmpenv.d.sha512); - break; -//LCOV_EXCL_START - case DIGEST_SHA3_256: /* FALLSTHROUGH */ - case DIGEST_SHA3_512: - default: - log_warn(LD_BUG, "Handling unexpected algorithm %d", digest->algorithm); - /* This is fatal, because it should never happen. */ - tor_assert_unreached(); - break; -//LCOV_EXCL_STOP - } - memcpy(out, r, out_len); - memwipe(r, 0, sizeof(r)); -} - -/** Allocate and return a new digest object with the same state as - * <b>digest</b> - */ -crypto_digest_t * -crypto_digest_dup(const crypto_digest_t *digest) -{ - tor_assert(digest); - const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm); - return tor_memdup(digest, alloc_bytes); -} - -/** Replace the state of the digest object <b>into</b> with the state - * of the digest object <b>from</b>. Requires that 'into' and 'from' - * have the same digest type. - */ -void -crypto_digest_assign(crypto_digest_t *into, - const crypto_digest_t *from) -{ - tor_assert(into); - tor_assert(from); - tor_assert(into->algorithm == from->algorithm); - const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm); - memcpy(into,from,alloc_bytes); -} - -/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest - * at <b>digest_out</b> to the hash of the concatenation of those strings, - * plus the optional string <b>append</b>, computed with the algorithm - * <b>alg</b>. - * <b>out_len</b> must be \<= DIGEST512_LEN. */ -void -crypto_digest_smartlist(char *digest_out, size_t len_out, - const smartlist_t *lst, - const char *append, - digest_algorithm_t alg) -{ - crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg); -} - -/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest - * at <b>digest_out</b> to the hash of the concatenation of: the - * optional string <b>prepend</b>, those strings, - * and the optional string <b>append</b>, computed with the algorithm - * <b>alg</b>. - * <b>len_out</b> must be \<= DIGEST512_LEN. */ -void -crypto_digest_smartlist_prefix(char *digest_out, size_t len_out, - const char *prepend, - const smartlist_t *lst, - const char *append, - digest_algorithm_t alg) -{ - crypto_digest_t *d = crypto_digest_new_internal(alg); - if (prepend) - crypto_digest_add_bytes(d, prepend, strlen(prepend)); - SMARTLIST_FOREACH(lst, const char *, cp, - crypto_digest_add_bytes(d, cp, strlen(cp))); - if (append) - crypto_digest_add_bytes(d, append, strlen(append)); - crypto_digest_get_digest(d, digest_out, len_out); - crypto_digest_free(d); -} - -/** Compute the HMAC-SHA-256 of the <b>msg_len</b> bytes in <b>msg</b>, using - * the <b>key</b> of length <b>key_len</b>. Store the DIGEST256_LEN-byte - * result in <b>hmac_out</b>. Asserts on failure. - */ -void -crypto_hmac_sha256(char *hmac_out, - const char *key, size_t key_len, - const char *msg, size_t msg_len) -{ - unsigned char *rv = NULL; - /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */ - tor_assert(key_len < INT_MAX); - tor_assert(msg_len < INT_MAX); - tor_assert(hmac_out); - rv = HMAC(EVP_sha256(), key, (int)key_len, (unsigned char*)msg, (int)msg_len, - (unsigned char*)hmac_out, NULL); - tor_assert(rv); -} - -/** Compute a MAC using SHA3-256 of <b>msg_len</b> bytes in <b>msg</b> using a - * <b>key</b> of length <b>key_len</b> and a <b>salt</b> of length - * <b>salt_len</b>. Store the result of <b>len_out</b> bytes in in - * <b>mac_out</b>. This function can't fail. */ -void -crypto_mac_sha3_256(uint8_t *mac_out, size_t len_out, - const uint8_t *key, size_t key_len, - const uint8_t *msg, size_t msg_len) -{ - crypto_digest_t *digest; - - const uint64_t key_len_netorder = tor_htonll(key_len); - - tor_assert(mac_out); - tor_assert(key); - tor_assert(msg); - - digest = crypto_digest256_new(DIGEST_SHA3_256); - - /* Order matters here that is any subsystem using this function should - * expect this very precise ordering in the MAC construction. */ - crypto_digest_add_bytes(digest, (const char *) &key_len_netorder, - sizeof(key_len_netorder)); - crypto_digest_add_bytes(digest, (const char *) key, key_len); - crypto_digest_add_bytes(digest, (const char *) msg, msg_len); - crypto_digest_get_digest(digest, (char *) mac_out, len_out); - crypto_digest_free(digest); -} - -/** Internal state for a eXtendable-Output Function (XOF). */ -struct crypto_xof_t { - keccak_state s; -}; - -/** Allocate a new XOF object backed by SHAKE-256. The security level - * provided is a function of the length of the output used. Read and - * understand FIPS-202 A.2 "Additional Consideration for Extendable-Output - * Functions" before using this construct. - */ -crypto_xof_t * -crypto_xof_new(void) -{ - crypto_xof_t *xof; - xof = tor_malloc(sizeof(crypto_xof_t)); - keccak_xof_init(&xof->s, 256); - return xof; -} - -/** Absorb bytes into a XOF object. Must not be called after a call to - * crypto_xof_squeeze_bytes() for the same instance, and will assert - * if attempted. - */ -void -crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len) -{ - int i = keccak_xof_absorb(&xof->s, data, len); - tor_assert(i == 0); -} - -/** Squeeze bytes out of a XOF object. Calling this routine will render - * the XOF instance ineligible to absorb further data. - */ -void -crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len) -{ - int i = keccak_xof_squeeze(&xof->s, out, len); - tor_assert(i == 0); -} - -/** Cleanse and deallocate a XOF object. */ -void -crypto_xof_free_(crypto_xof_t *xof) -{ - if (!xof) - return; - memwipe(xof, 0, sizeof(crypto_xof_t)); - tor_free(xof); -} - /* DH */ /** Our DH 'g' parameter */ @@ -1860,576 +1055,6 @@ crypto_dh_free_(crypto_dh_t *dh) tor_free(dh); } -/* random numbers */ - -/** How many bytes of entropy we add at once. - * - * This is how much entropy OpenSSL likes to add right now, so maybe it will - * work for us too. */ -#define ADD_ENTROPY 32 - -/** Set the seed of the weak RNG to a random value. */ -void -crypto_seed_weak_rng(tor_weak_rng_t *rng) -{ - unsigned seed; - crypto_rand((void*)&seed, sizeof(seed)); - tor_init_weak_random(rng, seed); -} - -#ifdef TOR_UNIT_TESTS -int break_strongest_rng_syscall = 0; -int break_strongest_rng_fallback = 0; -#endif - -/** Try to get <b>out_len</b> bytes of the strongest entropy we can generate, - * via system calls, storing it into <b>out</b>. Return 0 on success, -1 on - * failure. A maximum request size of 256 bytes is imposed. - */ -static int -crypto_strongest_rand_syscall(uint8_t *out, size_t out_len) -{ - tor_assert(out_len <= MAX_STRONGEST_RAND_SIZE); - - /* We only log at notice-level here because in the case that this function - * fails the crypto_strongest_rand_raw() caller will log with a warning-level - * message and let crypto_strongest_rand() error out and finally terminating - * Tor with an assertion error. - */ - -#ifdef TOR_UNIT_TESTS - if (break_strongest_rng_syscall) - return -1; -#endif - -#if defined(_WIN32) - static int provider_set = 0; - static HCRYPTPROV provider; - - if (!provider_set) { - if (!CryptAcquireContext(&provider, NULL, NULL, PROV_RSA_FULL, - CRYPT_VERIFYCONTEXT)) { - log_notice(LD_CRYPTO, "Unable to set Windows CryptoAPI provider [1]."); - return -1; - } - provider_set = 1; - } - if (!CryptGenRandom(provider, out_len, out)) { - log_notice(LD_CRYPTO, "Unable get entropy from the Windows CryptoAPI."); - return -1; - } - - return 0; -#elif defined(__linux__) && defined(SYS_getrandom) - static int getrandom_works = 1; /* Be optimistic about our chances... */ - - /* getrandom() isn't as straightforward as getentropy(), and has - * no glibc wrapper. - * - * As far as I can tell from getrandom(2) and the source code, the - * requests we issue will always succeed (though it will block on the - * call if /dev/urandom isn't seeded yet), since we are NOT specifying - * GRND_NONBLOCK and the request is <= 256 bytes. - * - * The manpage is unclear on what happens if a signal interrupts the call - * while the request is blocked due to lack of entropy.... - * - * We optimistically assume that getrandom() is available and functional - * because it is the way of the future, and 2 branch mispredicts pale in - * comparison to the overheads involved with failing to open - * /dev/srandom followed by opening and reading from /dev/urandom. - */ - if (PREDICT_LIKELY(getrandom_works)) { - long ret; - /* A flag of '0' here means to read from '/dev/urandom', and to - * block if insufficient entropy is available to service the - * request. - */ - const unsigned int flags = 0; - do { - ret = syscall(SYS_getrandom, out, out_len, flags); - } while (ret == -1 && ((errno == EINTR) ||(errno == EAGAIN))); - - if (PREDICT_UNLIKELY(ret == -1)) { - /* LCOV_EXCL_START we can't actually make the syscall fail in testing. */ - tor_assert(errno != EAGAIN); - tor_assert(errno != EINTR); - - /* Useful log message for errno. */ - if (errno == ENOSYS) { - log_notice(LD_CRYPTO, "Can't get entropy from getrandom()." - " You are running a version of Tor built to support" - " getrandom(), but the kernel doesn't implement this" - " function--probably because it is too old?" - " Trying fallback method instead."); - } else { - log_notice(LD_CRYPTO, "Can't get entropy from getrandom(): %s." - " Trying fallback method instead.", - strerror(errno)); - } - - getrandom_works = 0; /* Don't bother trying again. */ - return -1; - /* LCOV_EXCL_STOP */ - } - - tor_assert(ret == (long)out_len); - return 0; - } - - return -1; /* getrandom() previously failed unexpectedly. */ -#elif defined(HAVE_GETENTROPY) - /* getentropy() is what Linux's getrandom() wants to be when it grows up. - * the only gotcha is that requests are limited to 256 bytes. - */ - return getentropy(out, out_len); -#else - (void) out; -#endif /* defined(_WIN32) || ... */ - - /* This platform doesn't have a supported syscall based random. */ - return -1; -} - -/** Try to get <b>out_len</b> bytes of the strongest entropy we can generate, - * via the per-platform fallback mechanism, storing it into <b>out</b>. - * Return 0 on success, -1 on failure. A maximum request size of 256 bytes - * is imposed. - */ -static int -crypto_strongest_rand_fallback(uint8_t *out, size_t out_len) -{ -#ifdef TOR_UNIT_TESTS - if (break_strongest_rng_fallback) - return -1; -#endif - -#ifdef _WIN32 - /* Windows exclusively uses crypto_strongest_rand_syscall(). */ - (void)out; - (void)out_len; - return -1; -#else /* !(defined(_WIN32)) */ - static const char *filenames[] = { - "/dev/srandom", "/dev/urandom", "/dev/random", NULL - }; - int fd, i; - size_t n; - - for (i = 0; filenames[i]; ++i) { - log_debug(LD_FS, "Considering %s as entropy source", filenames[i]); - fd = open(sandbox_intern_string(filenames[i]), O_RDONLY, 0); - if (fd<0) continue; - log_info(LD_CRYPTO, "Reading entropy from \"%s\"", filenames[i]); - n = read_all(fd, (char*)out, out_len, 0); - close(fd); - if (n != out_len) { - /* LCOV_EXCL_START - * We can't make /dev/foorandom actually fail. */ - log_notice(LD_CRYPTO, - "Error reading from entropy source %s (read only %lu bytes).", - filenames[i], - (unsigned long)n); - return -1; - /* LCOV_EXCL_STOP */ - } - - return 0; - } - - return -1; -#endif /* defined(_WIN32) */ -} - -/** Try to get <b>out_len</b> bytes of the strongest entropy we can generate, - * storing it into <b>out</b>. Return 0 on success, -1 on failure. A maximum - * request size of 256 bytes is imposed. - */ -STATIC int -crypto_strongest_rand_raw(uint8_t *out, size_t out_len) -{ - static const size_t sanity_min_size = 16; - static const int max_attempts = 3; - tor_assert(out_len <= MAX_STRONGEST_RAND_SIZE); - - /* For buffers >= 16 bytes (128 bits), we sanity check the output by - * zero filling the buffer and ensuring that it actually was at least - * partially modified. - * - * Checking that any individual byte is non-zero seems like it would - * fail too often (p = out_len * 1/256) for comfort, but this is an - * "adjust according to taste" sort of check. - */ - memwipe(out, 0, out_len); - for (int i = 0; i < max_attempts; i++) { - /* Try to use the syscall/OS favored mechanism to get strong entropy. */ - if (crypto_strongest_rand_syscall(out, out_len) != 0) { - /* Try to use the less-favored mechanism to get strong entropy. */ - if (crypto_strongest_rand_fallback(out, out_len) != 0) { - /* Welp, we tried. Hopefully the calling code terminates the process - * since we're basically boned without good entropy. - */ - log_warn(LD_CRYPTO, - "Cannot get strong entropy: no entropy source found."); - return -1; - } - } - - if ((out_len < sanity_min_size) || !tor_mem_is_zero((char*)out, out_len)) - return 0; - } - - /* LCOV_EXCL_START - * - * We tried max_attempts times to fill a buffer >= 128 bits long, - * and each time it returned all '0's. Either the system entropy - * source is busted, or the user should go out and buy a ticket to - * every lottery on the planet. - */ - log_warn(LD_CRYPTO, "Strong OS entropy returned all zero buffer."); - - return -1; - /* LCOV_EXCL_STOP */ -} - -/** Try to get <b>out_len</b> bytes of the strongest entropy we can generate, - * storing it into <b>out</b>. - */ -void -crypto_strongest_rand(uint8_t *out, size_t out_len) -{ -#define DLEN SHA512_DIGEST_LENGTH - /* We're going to hash DLEN bytes from the system RNG together with some - * bytes from the openssl PRNG, in order to yield DLEN bytes. - */ - uint8_t inp[DLEN*2]; - uint8_t tmp[DLEN]; - tor_assert(out); - while (out_len) { - crypto_rand((char*) inp, DLEN); - if (crypto_strongest_rand_raw(inp+DLEN, DLEN) < 0) { - // LCOV_EXCL_START - log_err(LD_CRYPTO, "Failed to load strong entropy when generating an " - "important key. Exiting."); - /* Die with an assertion so we get a stack trace. */ - tor_assert(0); - // LCOV_EXCL_STOP - } - if (out_len >= DLEN) { - SHA512(inp, sizeof(inp), out); - out += DLEN; - out_len -= DLEN; - } else { - SHA512(inp, sizeof(inp), tmp); - memcpy(out, tmp, out_len); - break; - } - } - memwipe(tmp, 0, sizeof(tmp)); - memwipe(inp, 0, sizeof(inp)); -#undef DLEN -} - -/** Seed OpenSSL's random number generator with bytes from the operating - * system. Return 0 on success, -1 on failure. - */ -int -crypto_seed_rng(void) -{ - int rand_poll_ok = 0, load_entropy_ok = 0; - uint8_t buf[ADD_ENTROPY]; - - /* OpenSSL has a RAND_poll function that knows about more kinds of - * entropy than we do. We'll try calling that, *and* calling our own entropy - * functions. If one succeeds, we'll accept the RNG as seeded. */ - rand_poll_ok = RAND_poll(); - if (rand_poll_ok == 0) - log_warn(LD_CRYPTO, "RAND_poll() failed."); // LCOV_EXCL_LINE - - load_entropy_ok = !crypto_strongest_rand_raw(buf, sizeof(buf)); - if (load_entropy_ok) { - RAND_seed(buf, sizeof(buf)); - } - - memwipe(buf, 0, sizeof(buf)); - - if ((rand_poll_ok || load_entropy_ok) && RAND_status() == 1) - return 0; - else - return -1; -} - -/** Write <b>n</b> bytes of strong random data to <b>to</b>. Supports mocking - * for unit tests. - * - * This function is not allowed to fail; if it would fail to generate strong - * entropy, it must terminate the process instead. - */ -MOCK_IMPL(void, -crypto_rand, (char *to, size_t n)) -{ - crypto_rand_unmocked(to, n); -} - -/** Write <b>n</b> bytes of strong random data to <b>to</b>. Most callers - * will want crypto_rand instead. - * - * This function is not allowed to fail; if it would fail to generate strong - * entropy, it must terminate the process instead. - */ -void -crypto_rand_unmocked(char *to, size_t n) -{ - int r; - if (n == 0) - return; - - tor_assert(n < INT_MAX); - tor_assert(to); - r = RAND_bytes((unsigned char*)to, (int)n); - /* We consider a PRNG failure non-survivable. Let's assert so that we get a - * stack trace about where it happened. - */ - tor_assert(r >= 0); -} - -/** Return a pseudorandom integer, chosen uniformly from the values - * between 0 and <b>max</b>-1 inclusive. <b>max</b> must be between 1 and - * INT_MAX+1, inclusive. */ -int -crypto_rand_int(unsigned int max) -{ - unsigned int val; - unsigned int cutoff; - tor_assert(max <= ((unsigned int)INT_MAX)+1); - tor_assert(max > 0); /* don't div by 0 */ - - /* We ignore any values that are >= 'cutoff,' to avoid biasing the - * distribution with clipping at the upper end of unsigned int's - * range. - */ - cutoff = UINT_MAX - (UINT_MAX%max); - while (1) { - crypto_rand((char*)&val, sizeof(val)); - if (val < cutoff) - return val % max; - } -} - -/** Return a pseudorandom integer, chosen uniformly from the values i such - * that min <= i < max. - * - * <b>min</b> MUST be in range [0, <b>max</b>). - * <b>max</b> MUST be in range (min, INT_MAX]. - */ -int -crypto_rand_int_range(unsigned int min, unsigned int max) -{ - tor_assert(min < max); - tor_assert(max <= INT_MAX); - - /* The overflow is avoided here because crypto_rand_int() returns a value - * between 0 and (max - min) inclusive. */ - return min + crypto_rand_int(max - min); -} - -/** As crypto_rand_int_range, but supports uint64_t. */ -uint64_t -crypto_rand_uint64_range(uint64_t min, uint64_t max) -{ - tor_assert(min < max); - return min + crypto_rand_uint64(max - min); -} - -/** As crypto_rand_int_range, but supports time_t. */ -time_t -crypto_rand_time_range(time_t min, time_t max) -{ - tor_assert(min < max); - return min + (time_t)crypto_rand_uint64(max - min); -} - -/** Return a pseudorandom 64-bit integer, chosen uniformly from the values - * between 0 and <b>max</b>-1 inclusive. */ -uint64_t -crypto_rand_uint64(uint64_t max) -{ - uint64_t val; - uint64_t cutoff; - tor_assert(max < UINT64_MAX); - tor_assert(max > 0); /* don't div by 0 */ - - /* We ignore any values that are >= 'cutoff,' to avoid biasing the - * distribution with clipping at the upper end of unsigned int's - * range. - */ - cutoff = UINT64_MAX - (UINT64_MAX%max); - while (1) { - crypto_rand((char*)&val, sizeof(val)); - if (val < cutoff) - return val % max; - } -} - -/** Return a pseudorandom double d, chosen uniformly from the range - * 0.0 <= d < 1.0. - */ -double -crypto_rand_double(void) -{ - /* We just use an unsigned int here; we don't really care about getting - * more than 32 bits of resolution */ - unsigned int u; - crypto_rand((char*)&u, sizeof(u)); -#if SIZEOF_INT == 4 -#define UINT_MAX_AS_DOUBLE 4294967296.0 -#elif SIZEOF_INT == 8 -#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19 -#else -#error SIZEOF_INT is neither 4 nor 8 -#endif /* SIZEOF_INT == 4 || ... */ - return ((double)u) / UINT_MAX_AS_DOUBLE; -} - -/** Generate and return a new random hostname starting with <b>prefix</b>, - * ending with <b>suffix</b>, and containing no fewer than - * <b>min_rand_len</b> and no more than <b>max_rand_len</b> random base32 - * characters. Does not check for failure. - * - * Clip <b>max_rand_len</b> to MAX_DNS_LABEL_SIZE. - **/ -char * -crypto_random_hostname(int min_rand_len, int max_rand_len, const char *prefix, - const char *suffix) -{ - char *result, *rand_bytes; - int randlen, rand_bytes_len; - size_t resultlen, prefixlen; - - if (max_rand_len > MAX_DNS_LABEL_SIZE) - max_rand_len = MAX_DNS_LABEL_SIZE; - if (min_rand_len > max_rand_len) - min_rand_len = max_rand_len; - - randlen = crypto_rand_int_range(min_rand_len, max_rand_len+1); - - prefixlen = strlen(prefix); - resultlen = prefixlen + strlen(suffix) + randlen + 16; - - rand_bytes_len = ((randlen*5)+7)/8; - if (rand_bytes_len % 5) - rand_bytes_len += 5 - (rand_bytes_len%5); - rand_bytes = tor_malloc(rand_bytes_len); - crypto_rand(rand_bytes, rand_bytes_len); - - result = tor_malloc(resultlen); - memcpy(result, prefix, prefixlen); - base32_encode(result+prefixlen, resultlen-prefixlen, - rand_bytes, rand_bytes_len); - tor_free(rand_bytes); - strlcpy(result+prefixlen+randlen, suffix, resultlen-(prefixlen+randlen)); - - return result; -} - -/** Return a randomly chosen element of <b>sl</b>; or NULL if <b>sl</b> - * is empty. */ -void * -smartlist_choose(const smartlist_t *sl) -{ - int len = smartlist_len(sl); - if (len) - return smartlist_get(sl,crypto_rand_int(len)); - return NULL; /* no elements to choose from */ -} - -/** Scramble the elements of <b>sl</b> into a random order. */ -void -smartlist_shuffle(smartlist_t *sl) -{ - int i; - /* From the end of the list to the front, choose at random from the - positions we haven't looked at yet, and swap that position into the - current position. Remember to give "no swap" the same probability as - any other swap. */ - for (i = smartlist_len(sl)-1; i > 0; --i) { - int j = crypto_rand_int(i+1); - smartlist_swap(sl, i, j); - } -} - -/** - * Destroy the <b>sz</b> bytes of data stored at <b>mem</b>, setting them to - * the value <b>byte</b>. - * If <b>mem</b> is NULL or <b>sz</b> is zero, nothing happens. - * - * This function is preferable to memset, since many compilers will happily - * optimize out memset() when they can convince themselves that the data being - * cleared will never be read. - * - * Right now, our convention is to use this function when we are wiping data - * that's about to become inaccessible, such as stack buffers that are about - * to go out of scope or structures that are about to get freed. (In - * practice, it appears that the compilers we're currently using will optimize - * out the memset()s for stack-allocated buffers, but not those for - * about-to-be-freed structures. That could change, though, so we're being - * wary.) If there are live reads for the data, then you can just use - * memset(). - */ -void -memwipe(void *mem, uint8_t byte, size_t sz) -{ - if (sz == 0) { - return; - } - /* If sz is nonzero, then mem must not be NULL. */ - tor_assert(mem != NULL); - - /* Data this large is likely to be an underflow. */ - tor_assert(sz < SIZE_T_CEILING); - - /* Because whole-program-optimization exists, we may not be able to just - * have this function call "memset". A smart compiler could inline it, then - * eliminate dead memsets, and declare itself to be clever. */ - -#if defined(SecureZeroMemory) || defined(HAVE_SECUREZEROMEMORY) - /* Here's what you do on windows. */ - SecureZeroMemory(mem,sz); -#elif defined(HAVE_RTLSECUREZEROMEMORY) - RtlSecureZeroMemory(mem,sz); -#elif defined(HAVE_EXPLICIT_BZERO) - /* The BSDs provide this. */ - explicit_bzero(mem, sz); -#elif defined(HAVE_MEMSET_S) - /* This is in the C99 standard. */ - memset_s(mem, sz, 0, sz); -#else - /* This is a slow and ugly function from OpenSSL that fills 'mem' with junk - * based on the pointer value, then uses that junk to update a global - * variable. It's an elaborate ruse to trick the compiler into not - * optimizing out the "wipe this memory" code. Read it if you like zany - * programming tricks! In later versions of Tor, we should look for better - * not-optimized-out memory wiping stuff... - * - * ...or maybe not. In practice, there are pure-asm implementations of - * OPENSSL_cleanse() on most platforms, which ought to do the job. - **/ - - OPENSSL_cleanse(mem, sz); -#endif /* defined(SecureZeroMemory) || defined(HAVE_SECUREZEROMEMORY) || ... */ - - /* Just in case some caller of memwipe() is relying on getting a buffer - * filled with a particular value, fill the buffer. - * - * If this function gets inlined, this memset might get eliminated, but - * that's okay: We only care about this particular memset in the case where - * the caller should have been using memset(), and the memset() wouldn't get - * eliminated. In other words, this is here so that we won't break anything - * if somebody accidentally calls memwipe() instead of memset(). - **/ - memset(mem, byte, sz); -} - /** @{ */ /** Uninitialize the crypto library. Return 0 on success. Does not detect * failure. @@ -2437,11 +1062,15 @@ memwipe(void *mem, uint8_t byte, size_t sz) int crypto_global_cleanup(void) { +#ifndef OPENSSL_1_1_API EVP_cleanup(); +#endif #ifndef NEW_THREAD_API ERR_remove_thread_state(NULL); #endif +#ifndef OPENSSL_1_1_API ERR_free_strings(); +#endif if (dh_param_p) BN_clear_free(dh_param_p); @@ -2453,11 +1082,15 @@ crypto_global_cleanup(void) dh_param_p = dh_param_p_tls = dh_param_g = NULL; #ifndef DISABLE_ENGINES +#ifndef OPENSSL_1_1_API ENGINE_cleanup(); #endif +#endif CONF_modules_unload(1); +#ifndef OPENSSL_1_1_API CRYPTO_cleanup_all_ex_data(); +#endif crypto_openssl_free_all(); diff --git a/src/common/crypto.h b/src/common/crypto.h index a9c8837b9e..c773557310 100644 --- a/src/common/crypto.h +++ b/src/common/crypto.h @@ -17,20 +17,10 @@ #include <stdio.h> #include "torint.h" -#include "testsupport.h" #include "compat.h" #include "util.h" #include "crypto_rsa.h" -#include "keccak-tiny/keccak-tiny.h" - -/** Length of the output of our message digest. */ -#define DIGEST_LEN 20 -/** Length of the output of our second (improved) message digests. (For now - * this is just sha256, but it could be any other 256-bit digest.) */ -#define DIGEST256_LEN 32 -/** Length of the output of our 64-bit optimized message digests (SHA512). */ -#define DIGEST512_LEN 64 /** Length of our symmetric cipher's keys of 128-bit. */ #define CIPHER_KEY_LEN 16 /** Length of our symmetric cipher's IV of 128-bit. */ @@ -40,57 +30,15 @@ /** Length of our DH keys. */ #define DH_BYTES (1024/8) -/** Length of a sha1 message digest when encoded in base32 with trailing = - * signs removed. */ -#define BASE32_DIGEST_LEN 32 -/** Length of a sha1 message digest when encoded in base64 with trailing = - * signs removed. */ -#define BASE64_DIGEST_LEN 27 -/** Length of a sha256 message digest when encoded in base64 with trailing = - * signs removed. */ -#define BASE64_DIGEST256_LEN 43 -/** Length of a sha512 message digest when encoded in base64 with trailing = - * signs removed. */ -#define BASE64_DIGEST512_LEN 86 - /** Length of encoded public key fingerprints, including space; but not * including terminating NUL. */ #define FINGERPRINT_LEN 49 -/** Length of hex encoding of SHA1 digest, not including final NUL. */ -#define HEX_DIGEST_LEN 40 -/** Length of hex encoding of SHA256 digest, not including final NUL. */ -#define HEX_DIGEST256_LEN 64 -/** Length of hex encoding of SHA512 digest, not including final NUL. */ -#define HEX_DIGEST512_LEN 128 - -typedef enum { - DIGEST_SHA1 = 0, - DIGEST_SHA256 = 1, - DIGEST_SHA512 = 2, - DIGEST_SHA3_256 = 3, - DIGEST_SHA3_512 = 4, -} digest_algorithm_t; -#define N_DIGEST_ALGORITHMS (DIGEST_SHA3_512+1) -#define N_COMMON_DIGEST_ALGORITHMS (DIGEST_SHA256+1) - -/** A set of all the digests we commonly compute, taken on a single - * string. Any digests that are shorter than 512 bits are right-padded - * with 0 bits. - * - * Note that this representation wastes 44 bytes for the SHA1 case, so - * don't use it for anything where we need to allocate a whole bunch at - * once. - **/ -typedef struct { - char d[N_COMMON_DIGEST_ALGORITHMS][DIGEST256_LEN]; -} common_digests_t; typedef struct aes_cnt_cipher crypto_cipher_t; -typedef struct crypto_digest_t crypto_digest_t; -typedef struct crypto_xof_t crypto_xof_t; typedef struct crypto_dh_t crypto_dh_t; /* global state */ +int crypto_init_siphash_key(void); int crypto_early_init(void) ATTR_WUR; int crypto_global_init(int hardwareAccel, const char *accelName, @@ -114,24 +62,6 @@ void crypto_cipher_free_(crypto_cipher_t *env); #define crypto_cipher_free(c) \ FREE_AND_NULL(crypto_cipher_t, crypto_cipher_free_, (c)) -/* public key crypto */ -MOCK_DECL(int, crypto_pk_public_checksig_digest,(crypto_pk_t *env, - const char *data, size_t datalen, - const char *sig, size_t siglen)); -int crypto_pk_private_sign_digest(crypto_pk_t *env, char *to, size_t tolen, - const char *from, size_t fromlen); -int crypto_pk_obsolete_public_hybrid_encrypt(crypto_pk_t *env, char *to, - size_t tolen, - const char *from, size_t fromlen, - int padding, int force); -int crypto_pk_obsolete_private_hybrid_decrypt(crypto_pk_t *env, char *to, - size_t tolen, - const char *from, size_t fromlen, - int padding, int warnOnFailure); -int crypto_pk_get_digest(const crypto_pk_t *pk, char *digest_out); -int crypto_pk_get_common_digests(crypto_pk_t *pk, - common_digests_t *digests_out); - /* symmetric crypto */ const char *crypto_cipher_get_key(crypto_cipher_t *env); @@ -148,52 +78,6 @@ int crypto_cipher_decrypt_with_iv(const char *key, char *to, size_t tolen, const char *from, size_t fromlen); -/* SHA-1 and other digests. */ -int crypto_digest(char *digest, const char *m, size_t len); -int crypto_digest256(char *digest, const char *m, size_t len, - digest_algorithm_t algorithm); -int crypto_digest512(char *digest, const char *m, size_t len, - digest_algorithm_t algorithm); -int crypto_common_digests(common_digests_t *ds_out, const char *m, size_t len); -struct smartlist_t; -void crypto_digest_smartlist_prefix(char *digest_out, size_t len_out, - const char *prepend, - const struct smartlist_t *lst, - const char *append, - digest_algorithm_t alg); -void crypto_digest_smartlist(char *digest_out, size_t len_out, - const struct smartlist_t *lst, const char *append, - digest_algorithm_t alg); -const char *crypto_digest_algorithm_get_name(digest_algorithm_t alg); -size_t crypto_digest_algorithm_get_length(digest_algorithm_t alg); -int crypto_digest_algorithm_parse_name(const char *name); -crypto_digest_t *crypto_digest_new(void); -crypto_digest_t *crypto_digest256_new(digest_algorithm_t algorithm); -crypto_digest_t *crypto_digest512_new(digest_algorithm_t algorithm); -void crypto_digest_free_(crypto_digest_t *digest); -#define crypto_digest_free(d) \ - FREE_AND_NULL(crypto_digest_t, crypto_digest_free_, (d)) -void crypto_digest_add_bytes(crypto_digest_t *digest, const char *data, - size_t len); -void crypto_digest_get_digest(crypto_digest_t *digest, - char *out, size_t out_len); -crypto_digest_t *crypto_digest_dup(const crypto_digest_t *digest); -void crypto_digest_assign(crypto_digest_t *into, - const crypto_digest_t *from); -void crypto_hmac_sha256(char *hmac_out, - const char *key, size_t key_len, - const char *msg, size_t msg_len); -void crypto_mac_sha3_256(uint8_t *mac_out, size_t len_out, - const uint8_t *key, size_t key_len, - const uint8_t *msg, size_t msg_len); - -crypto_xof_t *crypto_xof_new(void); -void crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len); -void crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len); -void crypto_xof_free_(crypto_xof_t *xof); -#define crypto_xof_free(xof) \ - FREE_AND_NULL(crypto_xof_t, crypto_xof_free_, (xof)) - /* Key negotiation */ #define DH_TYPE_CIRCUIT 1 #define DH_TYPE_REND 2 @@ -219,31 +103,6 @@ int crypto_expand_key_material_rfc5869_sha256( const uint8_t *info_in, size_t info_in_len, uint8_t *key_out, size_t key_out_len); -/* random numbers */ -int crypto_seed_rng(void) ATTR_WUR; -MOCK_DECL(void,crypto_rand,(char *to, size_t n)); -void crypto_rand_unmocked(char *to, size_t n); -void crypto_strongest_rand(uint8_t *out, size_t out_len); -int crypto_rand_int(unsigned int max); -int crypto_rand_int_range(unsigned int min, unsigned int max); -uint64_t crypto_rand_uint64_range(uint64_t min, uint64_t max); -time_t crypto_rand_time_range(time_t min, time_t max); -uint64_t crypto_rand_uint64(uint64_t max); -double crypto_rand_double(void); -struct tor_weak_rng_t; -void crypto_seed_weak_rng(struct tor_weak_rng_t *rng); -int crypto_init_siphash_key(void); - -char *crypto_random_hostname(int min_rand_len, int max_rand_len, - const char *prefix, const char *suffix); - -struct smartlist_t; -void *smartlist_choose(const struct smartlist_t *sl); -void smartlist_shuffle(struct smartlist_t *sl); - -/** OpenSSL-based utility functions. */ -void memwipe(void *mem, uint8_t byte, size_t sz); - /* Prototypes for private functions only used by tortls.c, crypto.c, and the * unit tests. */ struct dh_st; @@ -251,20 +110,5 @@ struct dh_st *crypto_dh_get_dh_(crypto_dh_t *dh); void crypto_add_spaces_to_fp(char *out, size_t outlen, const char *in); -#ifdef CRYPTO_PRIVATE - -STATIC int crypto_force_rand_ssleay(void); -STATIC int crypto_strongest_rand_raw(uint8_t *out, size_t out_len); - -#ifdef TOR_UNIT_TESTS -extern int break_strongest_rng_syscall; -extern int break_strongest_rng_fallback; -#endif -#endif /* defined(CRYPTO_PRIVATE) */ - -#ifdef TOR_UNIT_TESTS -digest_algorithm_t crypto_digest_get_algorithm(crypto_digest_t *digest); -#endif - #endif /* !defined(TOR_CRYPTO_H) */ diff --git a/src/common/crypto_curve25519.c b/src/common/crypto_curve25519.c index 8793fa6274..996d94c6e2 100644 --- a/src/common/crypto_curve25519.c +++ b/src/common/crypto_curve25519.c @@ -21,9 +21,11 @@ #include <sys/stat.h> #endif #include "container.h" -#include "crypto.h" #include "crypto_curve25519.h" +#include "crypto_digest.h" #include "crypto_format.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "util.h" #include "torlog.h" diff --git a/src/common/crypto_curve25519.h b/src/common/crypto_curve25519.h index 11f7423b07..4834fa0836 100644 --- a/src/common/crypto_curve25519.h +++ b/src/common/crypto_curve25519.h @@ -6,6 +6,7 @@ #include "testsupport.h" #include "torint.h" +#include "crypto_digest.h" #include "crypto_openssl_mgt.h" /** Length of a curve25519 public key when encoded. */ diff --git a/src/common/crypto_digest.c b/src/common/crypto_digest.c new file mode 100644 index 0000000000..9f9a1a1e2c --- /dev/null +++ b/src/common/crypto_digest.c @@ -0,0 +1,583 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_digest.c + * \brief Block of functions related with digest and xof utilities and + * operations. + **/ + +#include "container.h" +#include "crypto_digest.h" +#include "crypto_openssl_mgt.h" +#include "crypto_util.h" +#include "torlog.h" + +#include "keccak-tiny/keccak-tiny.h" + +DISABLE_GCC_WARNING(redundant-decls) + +#include <openssl/hmac.h> +#include <openssl/sha.h> + +ENABLE_GCC_WARNING(redundant-decls) + +/* Crypto digest functions */ + +/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in + * <b>m</b>. Write the DIGEST_LEN byte result into <b>digest</b>. + * Return 0 on success, -1 on failure. + */ +int +crypto_digest(char *digest, const char *m, size_t len) +{ + tor_assert(m); + tor_assert(digest); + if (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL) + return -1; + return 0; +} + +/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>, + * using the algorithm <b>algorithm</b>. Write the DIGEST_LEN256-byte result + * into <b>digest</b>. Return 0 on success, -1 on failure. */ +int +crypto_digest256(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm) +{ + tor_assert(m); + tor_assert(digest); + tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); + + int ret = 0; + if (algorithm == DIGEST_SHA256) + ret = (SHA256((const uint8_t*)m,len,(uint8_t*)digest) != NULL); + else + ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len) + > -1); + + if (!ret) + return -1; + return 0; +} + +/** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>, + * using the algorithm <b>algorithm</b>. Write the DIGEST_LEN512-byte result + * into <b>digest</b>. Return 0 on success, -1 on failure. */ +int +crypto_digest512(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm) +{ + tor_assert(m); + tor_assert(digest); + tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); + + int ret = 0; + if (algorithm == DIGEST_SHA512) + ret = (SHA512((const unsigned char*)m,len,(unsigned char*)digest) + != NULL); + else + ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len) + > -1); + + if (!ret) + return -1; + return 0; +} + +/** Set the common_digests_t in <b>ds_out</b> to contain every digest on the + * <b>len</b> bytes in <b>m</b> that we know how to compute. Return 0 on + * success, -1 on failure. */ +int +crypto_common_digests(common_digests_t *ds_out, const char *m, size_t len) +{ + tor_assert(ds_out); + memset(ds_out, 0, sizeof(*ds_out)); + if (crypto_digest(ds_out->d[DIGEST_SHA1], m, len) < 0) + return -1; + if (crypto_digest256(ds_out->d[DIGEST_SHA256], m, len, DIGEST_SHA256) < 0) + return -1; + + return 0; +} + +/** Return the name of an algorithm, as used in directory documents. */ +const char * +crypto_digest_algorithm_get_name(digest_algorithm_t alg) +{ + switch (alg) { + case DIGEST_SHA1: + return "sha1"; + case DIGEST_SHA256: + return "sha256"; + case DIGEST_SHA512: + return "sha512"; + case DIGEST_SHA3_256: + return "sha3-256"; + case DIGEST_SHA3_512: + return "sha3-512"; + // LCOV_EXCL_START + default: + tor_fragile_assert(); + return "??unknown_digest??"; + // LCOV_EXCL_STOP + } +} + +/** Given the name of a digest algorithm, return its integer value, or -1 if + * the name is not recognized. */ +int +crypto_digest_algorithm_parse_name(const char *name) +{ + if (!strcmp(name, "sha1")) + return DIGEST_SHA1; + else if (!strcmp(name, "sha256")) + return DIGEST_SHA256; + else if (!strcmp(name, "sha512")) + return DIGEST_SHA512; + else if (!strcmp(name, "sha3-256")) + return DIGEST_SHA3_256; + else if (!strcmp(name, "sha3-512")) + return DIGEST_SHA3_512; + else + return -1; +} + +/** Given an algorithm, return the digest length in bytes. */ +size_t +crypto_digest_algorithm_get_length(digest_algorithm_t alg) +{ + switch (alg) { + case DIGEST_SHA1: + return DIGEST_LEN; + case DIGEST_SHA256: + return DIGEST256_LEN; + case DIGEST_SHA512: + return DIGEST512_LEN; + case DIGEST_SHA3_256: + return DIGEST256_LEN; + case DIGEST_SHA3_512: + return DIGEST512_LEN; + default: + tor_assert(0); // LCOV_EXCL_LINE + return 0; /* Unreachable */ // LCOV_EXCL_LINE + } +} + +/** Intermediate information about the digest of a stream of data. */ +struct crypto_digest_t { + digest_algorithm_t algorithm; /**< Which algorithm is in use? */ + /** State for the digest we're using. Only one member of the + * union is usable, depending on the value of <b>algorithm</b>. Note also + * that space for other members might not even be allocated! + */ + union { + SHA_CTX sha1; /**< state for SHA1 */ + SHA256_CTX sha2; /**< state for SHA256 */ + SHA512_CTX sha512; /**< state for SHA512 */ + keccak_state sha3; /**< state for SHA3-[256,512] */ + } d; +}; + +#ifdef TOR_UNIT_TESTS + +digest_algorithm_t +crypto_digest_get_algorithm(crypto_digest_t *digest) +{ + tor_assert(digest); + + return digest->algorithm; +} + +#endif /* defined(TOR_UNIT_TESTS) */ + +/** + * Return the number of bytes we need to malloc in order to get a + * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe + * when we free one. + */ +static size_t +crypto_digest_alloc_bytes(digest_algorithm_t alg) +{ + /* Helper: returns the number of bytes in the 'f' field of 'st' */ +#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f )) + /* Gives the length of crypto_digest_t through the end of the field 'd' */ +#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \ + STRUCT_FIELD_SIZE(crypto_digest_t, f)) + switch (alg) { + case DIGEST_SHA1: + return END_OF_FIELD(d.sha1); + case DIGEST_SHA256: + return END_OF_FIELD(d.sha2); + case DIGEST_SHA512: + return END_OF_FIELD(d.sha512); + case DIGEST_SHA3_256: + case DIGEST_SHA3_512: + return END_OF_FIELD(d.sha3); + default: + tor_assert(0); // LCOV_EXCL_LINE + return 0; // LCOV_EXCL_LINE + } +#undef END_OF_FIELD +#undef STRUCT_FIELD_SIZE +} + +/** + * Internal function: create and return a new digest object for 'algorithm'. + * Does not typecheck the algorithm. + */ +static crypto_digest_t * +crypto_digest_new_internal(digest_algorithm_t algorithm) +{ + crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm)); + r->algorithm = algorithm; + + switch (algorithm) + { + case DIGEST_SHA1: + SHA1_Init(&r->d.sha1); + break; + case DIGEST_SHA256: + SHA256_Init(&r->d.sha2); + break; + case DIGEST_SHA512: + SHA512_Init(&r->d.sha512); + break; + case DIGEST_SHA3_256: + keccak_digest_init(&r->d.sha3, 256); + break; + case DIGEST_SHA3_512: + keccak_digest_init(&r->d.sha3, 512); + break; + default: + tor_assert_unreached(); + } + + return r; +} + +/** Allocate and return a new digest object to compute SHA1 digests. + */ +crypto_digest_t * +crypto_digest_new(void) +{ + return crypto_digest_new_internal(DIGEST_SHA1); +} + +/** Allocate and return a new digest object to compute 256-bit digests + * using <b>algorithm</b>. + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest256_new` + * C_RUST_COUPLED: `crypto::digest::Sha256::default` + */ +crypto_digest_t * +crypto_digest256_new(digest_algorithm_t algorithm) +{ + tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256); + return crypto_digest_new_internal(algorithm); +} + +/** Allocate and return a new digest object to compute 512-bit digests + * using <b>algorithm</b>. */ +crypto_digest_t * +crypto_digest512_new(digest_algorithm_t algorithm) +{ + tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512); + return crypto_digest_new_internal(algorithm); +} + +/** Deallocate a digest object. + */ +void +crypto_digest_free_(crypto_digest_t *digest) +{ + if (!digest) + return; + size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); + memwipe(digest, 0, bytes); + tor_free(digest); +} + +/** Add <b>len</b> bytes from <b>data</b> to the digest object. + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_add_bytess` + * C_RUST_COUPLED: `crypto::digest::Sha256::process` + */ +void +crypto_digest_add_bytes(crypto_digest_t *digest, const char *data, + size_t len) +{ + tor_assert(digest); + tor_assert(data); + /* Using the SHA*_*() calls directly means we don't support doing + * SHA in hardware. But so far the delay of getting the question + * to the hardware, and hearing the answer, is likely higher than + * just doing it ourselves. Hashes are fast. + */ + switch (digest->algorithm) { + case DIGEST_SHA1: + SHA1_Update(&digest->d.sha1, (void*)data, len); + break; + case DIGEST_SHA256: + SHA256_Update(&digest->d.sha2, (void*)data, len); + break; + case DIGEST_SHA512: + SHA512_Update(&digest->d.sha512, (void*)data, len); + break; + case DIGEST_SHA3_256: /* FALLSTHROUGH */ + case DIGEST_SHA3_512: + keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len); + break; + default: + /* LCOV_EXCL_START */ + tor_fragile_assert(); + break; + /* LCOV_EXCL_STOP */ + } +} + +/** Compute the hash of the data that has been passed to the digest + * object; write the first out_len bytes of the result to <b>out</b>. + * <b>out_len</b> must be \<= DIGEST512_LEN. + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_get_digest` + * C_RUST_COUPLED: `impl digest::FixedOutput for Sha256` + */ +void +crypto_digest_get_digest(crypto_digest_t *digest, + char *out, size_t out_len) +{ + unsigned char r[DIGEST512_LEN]; + crypto_digest_t tmpenv; + tor_assert(digest); + tor_assert(out); + tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm)); + + /* The SHA-3 code handles copying into a temporary ctx, and also can handle + * short output buffers by truncating appropriately. */ + if (digest->algorithm == DIGEST_SHA3_256 || + digest->algorithm == DIGEST_SHA3_512) { + keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len); + return; + } + + const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm); + /* memcpy into a temporary ctx, since SHA*_Final clears the context */ + memcpy(&tmpenv, digest, alloc_bytes); + switch (digest->algorithm) { + case DIGEST_SHA1: + SHA1_Final(r, &tmpenv.d.sha1); + break; + case DIGEST_SHA256: + SHA256_Final(r, &tmpenv.d.sha2); + break; + case DIGEST_SHA512: + SHA512_Final(r, &tmpenv.d.sha512); + break; +//LCOV_EXCL_START + case DIGEST_SHA3_256: /* FALLSTHROUGH */ + case DIGEST_SHA3_512: + default: + log_warn(LD_BUG, "Handling unexpected algorithm %d", digest->algorithm); + /* This is fatal, because it should never happen. */ + tor_assert_unreached(); + break; +//LCOV_EXCL_STOP + } + memcpy(out, r, out_len); + memwipe(r, 0, sizeof(r)); +} + +/** Allocate and return a new digest object with the same state as + * <b>digest</b> + * + * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_dup` + * C_RUST_COUPLED: `impl Clone for crypto::digest::Sha256` + */ +crypto_digest_t * +crypto_digest_dup(const crypto_digest_t *digest) +{ + tor_assert(digest); + const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm); + return tor_memdup(digest, alloc_bytes); +} + +/** Temporarily save the state of <b>digest</b> in <b>checkpoint</b>. + * Asserts that <b>digest</b> is a SHA1 digest object. + */ +void +crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint, + const crypto_digest_t *digest) +{ + const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); + tor_assert(bytes <= sizeof(checkpoint->mem)); + memcpy(checkpoint->mem, digest, bytes); +} + +/** Restore the state of <b>digest</b> from <b>checkpoint</b>. + * Asserts that <b>digest</b> is a SHA1 digest object. Requires that the + * state was previously stored with crypto_digest_checkpoint() */ +void +crypto_digest_restore(crypto_digest_t *digest, + const crypto_digest_checkpoint_t *checkpoint) +{ + const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm); + memcpy(digest, checkpoint->mem, bytes); +} + +/** Replace the state of the digest object <b>into</b> with the state + * of the digest object <b>from</b>. Requires that 'into' and 'from' + * have the same digest type. + */ +void +crypto_digest_assign(crypto_digest_t *into, + const crypto_digest_t *from) +{ + tor_assert(into); + tor_assert(from); + tor_assert(into->algorithm == from->algorithm); + const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm); + memcpy(into,from,alloc_bytes); +} + +/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest + * at <b>digest_out</b> to the hash of the concatenation of those strings, + * plus the optional string <b>append</b>, computed with the algorithm + * <b>alg</b>. + * <b>out_len</b> must be \<= DIGEST512_LEN. */ +void +crypto_digest_smartlist(char *digest_out, size_t len_out, + const smartlist_t *lst, + const char *append, + digest_algorithm_t alg) +{ + crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg); +} + +/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest + * at <b>digest_out</b> to the hash of the concatenation of: the + * optional string <b>prepend</b>, those strings, + * and the optional string <b>append</b>, computed with the algorithm + * <b>alg</b>. + * <b>len_out</b> must be \<= DIGEST512_LEN. */ +void +crypto_digest_smartlist_prefix(char *digest_out, size_t len_out, + const char *prepend, + const smartlist_t *lst, + const char *append, + digest_algorithm_t alg) +{ + crypto_digest_t *d = crypto_digest_new_internal(alg); + if (prepend) + crypto_digest_add_bytes(d, prepend, strlen(prepend)); + SMARTLIST_FOREACH(lst, const char *, cp, + crypto_digest_add_bytes(d, cp, strlen(cp))); + if (append) + crypto_digest_add_bytes(d, append, strlen(append)); + crypto_digest_get_digest(d, digest_out, len_out); + crypto_digest_free(d); +} + +/** Compute the HMAC-SHA-256 of the <b>msg_len</b> bytes in <b>msg</b>, using + * the <b>key</b> of length <b>key_len</b>. Store the DIGEST256_LEN-byte + * result in <b>hmac_out</b>. Asserts on failure. + */ +void +crypto_hmac_sha256(char *hmac_out, + const char *key, size_t key_len, + const char *msg, size_t msg_len) +{ + unsigned char *rv = NULL; + /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */ + tor_assert(key_len < INT_MAX); + tor_assert(msg_len < INT_MAX); + tor_assert(hmac_out); + rv = HMAC(EVP_sha256(), key, (int)key_len, (unsigned char*)msg, (int)msg_len, + (unsigned char*)hmac_out, NULL); + tor_assert(rv); +} + +/** Compute a MAC using SHA3-256 of <b>msg_len</b> bytes in <b>msg</b> using a + * <b>key</b> of length <b>key_len</b> and a <b>salt</b> of length + * <b>salt_len</b>. Store the result of <b>len_out</b> bytes in in + * <b>mac_out</b>. This function can't fail. */ +void +crypto_mac_sha3_256(uint8_t *mac_out, size_t len_out, + const uint8_t *key, size_t key_len, + const uint8_t *msg, size_t msg_len) +{ + crypto_digest_t *digest; + + const uint64_t key_len_netorder = tor_htonll(key_len); + + tor_assert(mac_out); + tor_assert(key); + tor_assert(msg); + + digest = crypto_digest256_new(DIGEST_SHA3_256); + + /* Order matters here that is any subsystem using this function should + * expect this very precise ordering in the MAC construction. */ + crypto_digest_add_bytes(digest, (const char *) &key_len_netorder, + sizeof(key_len_netorder)); + crypto_digest_add_bytes(digest, (const char *) key, key_len); + crypto_digest_add_bytes(digest, (const char *) msg, msg_len); + crypto_digest_get_digest(digest, (char *) mac_out, len_out); + crypto_digest_free(digest); +} + +/* xof functions */ + +/** Internal state for a eXtendable-Output Function (XOF). */ +struct crypto_xof_t { + keccak_state s; +}; + +/** Allocate a new XOF object backed by SHAKE-256. The security level + * provided is a function of the length of the output used. Read and + * understand FIPS-202 A.2 "Additional Consideration for Extendable-Output + * Functions" before using this construct. + */ +crypto_xof_t * +crypto_xof_new(void) +{ + crypto_xof_t *xof; + xof = tor_malloc(sizeof(crypto_xof_t)); + keccak_xof_init(&xof->s, 256); + return xof; +} + +/** Absorb bytes into a XOF object. Must not be called after a call to + * crypto_xof_squeeze_bytes() for the same instance, and will assert + * if attempted. + */ +void +crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len) +{ + int i = keccak_xof_absorb(&xof->s, data, len); + tor_assert(i == 0); +} + +/** Squeeze bytes out of a XOF object. Calling this routine will render + * the XOF instance ineligible to absorb further data. + */ +void +crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len) +{ + int i = keccak_xof_squeeze(&xof->s, out, len); + tor_assert(i == 0); +} + +/** Cleanse and deallocate a XOF object. */ +void +crypto_xof_free_(crypto_xof_t *xof) +{ + if (!xof) + return; + memwipe(xof, 0, sizeof(crypto_xof_t)); + tor_free(xof); +} + diff --git a/src/common/crypto_digest.h b/src/common/crypto_digest.h new file mode 100644 index 0000000000..3bd74acdfa --- /dev/null +++ b/src/common/crypto_digest.h @@ -0,0 +1,136 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_digest.h + * + * \brief Headers for crypto_digest.c + **/ + +#ifndef TOR_CRYPTO_DIGEST_H +#define TOR_CRYPTO_DIGEST_H + +#include <stdio.h> + +#include "container.h" +#include "torint.h" + +/** Length of the output of our message digest. */ +#define DIGEST_LEN 20 +/** Length of the output of our second (improved) message digests. (For now + * this is just sha256, but it could be any other 256-bit digest.) */ +#define DIGEST256_LEN 32 +/** Length of the output of our 64-bit optimized message digests (SHA512). */ +#define DIGEST512_LEN 64 + +/** Length of a sha1 message digest when encoded in base32 with trailing = + * signs removed. */ +#define BASE32_DIGEST_LEN 32 +/** Length of a sha1 message digest when encoded in base64 with trailing = + * signs removed. */ +#define BASE64_DIGEST_LEN 27 +/** Length of a sha256 message digest when encoded in base64 with trailing = + * signs removed. */ +#define BASE64_DIGEST256_LEN 43 +/** Length of a sha512 message digest when encoded in base64 with trailing = + * signs removed. */ +#define BASE64_DIGEST512_LEN 86 + +/** Length of hex encoding of SHA1 digest, not including final NUL. */ +#define HEX_DIGEST_LEN 40 +/** Length of hex encoding of SHA256 digest, not including final NUL. */ +#define HEX_DIGEST256_LEN 64 +/** Length of hex encoding of SHA512 digest, not including final NUL. */ +#define HEX_DIGEST512_LEN 128 + +typedef enum { + DIGEST_SHA1 = 0, + DIGEST_SHA256 = 1, + DIGEST_SHA512 = 2, + DIGEST_SHA3_256 = 3, + DIGEST_SHA3_512 = 4, +} digest_algorithm_t; +#define N_DIGEST_ALGORITHMS (DIGEST_SHA3_512+1) +#define N_COMMON_DIGEST_ALGORITHMS (DIGEST_SHA256+1) + +#define DIGEST_CHECKPOINT_BYTES (SIZEOF_VOID_P + 512) +/** Structure used to temporarily save the a digest object. Only implemented + * for SHA1 digest for now. */ +typedef struct crypto_digest_checkpoint_t { + uint8_t mem[DIGEST_CHECKPOINT_BYTES]; +} crypto_digest_checkpoint_t; + +/** A set of all the digests we commonly compute, taken on a single + * string. Any digests that are shorter than 512 bits are right-padded + * with 0 bits. + * + * Note that this representation wastes 44 bytes for the SHA1 case, so + * don't use it for anything where we need to allocate a whole bunch at + * once. + **/ +typedef struct { + char d[N_COMMON_DIGEST_ALGORITHMS][DIGEST256_LEN]; +} common_digests_t; + +typedef struct crypto_digest_t crypto_digest_t; +typedef struct crypto_xof_t crypto_xof_t; + +/* SHA-1 and other digests */ +int crypto_digest(char *digest, const char *m, size_t len); +int crypto_digest256(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm); +int crypto_digest512(char *digest, const char *m, size_t len, + digest_algorithm_t algorithm); +int crypto_common_digests(common_digests_t *ds_out, const char *m, size_t len); +void crypto_digest_smartlist_prefix(char *digest_out, size_t len_out, + const char *prepend, + const struct smartlist_t *lst, + const char *append, + digest_algorithm_t alg); +void crypto_digest_smartlist(char *digest_out, size_t len_out, + const struct smartlist_t *lst, const char *append, + digest_algorithm_t alg); +const char *crypto_digest_algorithm_get_name(digest_algorithm_t alg); +size_t crypto_digest_algorithm_get_length(digest_algorithm_t alg); +int crypto_digest_algorithm_parse_name(const char *name); +crypto_digest_t *crypto_digest_new(void); +crypto_digest_t *crypto_digest256_new(digest_algorithm_t algorithm); +crypto_digest_t *crypto_digest512_new(digest_algorithm_t algorithm); +void crypto_digest_free_(crypto_digest_t *digest); +#define crypto_digest_free(d) \ + FREE_AND_NULL(crypto_digest_t, crypto_digest_free_, (d)) +void crypto_digest_add_bytes(crypto_digest_t *digest, const char *data, + size_t len); +void crypto_digest_get_digest(crypto_digest_t *digest, + char *out, size_t out_len); +crypto_digest_t *crypto_digest_dup(const crypto_digest_t *digest); +void crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint, + const crypto_digest_t *digest); +void crypto_digest_restore(crypto_digest_t *digest, + const crypto_digest_checkpoint_t *checkpoint); +void crypto_digest_assign(crypto_digest_t *into, + const crypto_digest_t *from); +void crypto_hmac_sha256(char *hmac_out, + const char *key, size_t key_len, + const char *msg, size_t msg_len); +void crypto_mac_sha3_256(uint8_t *mac_out, size_t len_out, + const uint8_t *key, size_t key_len, + const uint8_t *msg, size_t msg_len); + +/* xof functions*/ +crypto_xof_t *crypto_xof_new(void); +void crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len); +void crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len); +void crypto_xof_free_(crypto_xof_t *xof); +#define crypto_xof_free(xof) \ + FREE_AND_NULL(crypto_xof_t, crypto_xof_free_, (xof)) + +#ifdef TOR_UNIT_TESTS +digest_algorithm_t crypto_digest_get_algorithm(crypto_digest_t *digest); +#endif + +#endif /* !defined(TOR_CRYPTO_DIGEST_H) */ + diff --git a/src/common/crypto_ed25519.c b/src/common/crypto_ed25519.c index b962a59de1..9c13e3bdf0 100644 --- a/src/common/crypto_ed25519.c +++ b/src/common/crypto_ed25519.c @@ -21,11 +21,12 @@ #include <sys/stat.h> #endif -#include "crypto.h" - #include "crypto_curve25519.h" +#include "crypto_digest.h" #include "crypto_ed25519.h" #include "crypto_format.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "torlog.h" #include "util.h" #include "util_format.h" diff --git a/src/common/crypto_format.c b/src/common/crypto_format.c index 1d090a8770..460e85bac1 100644 --- a/src/common/crypto_format.c +++ b/src/common/crypto_format.c @@ -15,10 +15,11 @@ #include <sys/stat.h> #endif #include "container.h" -#include "crypto.h" #include "crypto_curve25519.h" +#include "crypto_digest.h" #include "crypto_ed25519.h" #include "crypto_format.h" +#include "crypto_util.h" #include "util.h" #include "util_format.h" #include "torlog.h" diff --git a/src/common/crypto_pwbox.c b/src/common/crypto_pwbox.c index 12acc9331c..c2bd1d26cb 100644 --- a/src/common/crypto_pwbox.c +++ b/src/common/crypto_pwbox.c @@ -9,8 +9,11 @@ */ #include "crypto.h" -#include "crypto_s2k.h" +#include "crypto_digest.h" #include "crypto_pwbox.h" +#include "crypto_rand.h" +#include "crypto_s2k.h" +#include "crypto_util.h" #include "di_ops.h" #include "util.h" #include "pwbox.h" diff --git a/src/common/crypto_rand.c b/src/common/crypto_rand.c new file mode 100644 index 0000000000..df2e2f65d3 --- /dev/null +++ b/src/common/crypto_rand.c @@ -0,0 +1,615 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_rand.c + * + * \brief Functions for initialising and seeding (pseudo-)random + * number generators, and working with randomness. + **/ + +#ifndef CRYPTO_RAND_PRIVATE +#define CRYPTO_RAND_PRIVATE + +#include "crypto_rand.h" + +#ifdef _WIN32 +#include <windows.h> +#include <wincrypt.h> +#endif /* defined(_WIN32) */ + +#include "container.h" +#include "compat.h" +#include "compat_openssl.h" +#include "crypto_util.h" +#include "sandbox.h" +#include "testsupport.h" +#include "torlog.h" +#include "util.h" +#include "util_format.h" + +DISABLE_GCC_WARNING(redundant-decls) +#include <openssl/rand.h> +ENABLE_GCC_WARNING(redundant-decls) + +#if __GNUC__ && GCC_VERSION >= 402 +#if GCC_VERSION >= 406 +#pragma GCC diagnostic pop +#else +#pragma GCC diagnostic warning "-Wredundant-decls" +#endif +#endif /* __GNUC__ && GCC_VERSION >= 402 */ + +#ifdef HAVE_FCNTL_H +#include <fcntl.h> +#endif +#ifdef HAVE_SYS_FCNTL_H +#include <sys/fcntl.h> +#endif +#ifdef HAVE_SYS_STAT_H +#include <sys/stat.h> +#endif +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif +#ifdef HAVE_SYS_SYSCALL_H +#include <sys/syscall.h> +#endif +#ifdef HAVE_SYS_RANDOM_H +#include <sys/random.h> +#endif + +/** + * How many bytes of entropy we add at once. + * + * This is how much entropy OpenSSL likes to add right now, so maybe it will + * work for us too. + **/ +#define ADD_ENTROPY 32 + +/** + * Longest recognized DNS query. + **/ +#define MAX_DNS_LABEL_SIZE 63 + +/** + * Largest strong entropy request permitted. + **/ +#define MAX_STRONGEST_RAND_SIZE 256 + +/** + * Set the seed of the weak RNG to a random value. + **/ +void +crypto_seed_weak_rng(tor_weak_rng_t *rng) +{ + unsigned seed; + crypto_rand((void*)&seed, sizeof(seed)); + tor_init_weak_random(rng, seed); +} + +#ifdef TOR_UNIT_TESTS +int break_strongest_rng_syscall = 0; +int break_strongest_rng_fallback = 0; +#endif + +/** + * Try to get <b>out_len</b> bytes of the strongest entropy we can generate, + * via system calls, storing it into <b>out</b>. Return 0 on success, -1 on + * failure. A maximum request size of 256 bytes is imposed. + **/ +static int +crypto_strongest_rand_syscall(uint8_t *out, size_t out_len) +{ + tor_assert(out_len <= MAX_STRONGEST_RAND_SIZE); + + /* We only log at notice-level here because in the case that this function + * fails the crypto_strongest_rand_raw() caller will log with a warning-level + * message and let crypto_strongest_rand() error out and finally terminating + * Tor with an assertion error. + */ + +#ifdef TOR_UNIT_TESTS + if (break_strongest_rng_syscall) + return -1; +#endif + +#if defined(_WIN32) + static int provider_set = 0; + static HCRYPTPROV provider; + + if (!provider_set) { + if (!CryptAcquireContext(&provider, NULL, NULL, PROV_RSA_FULL, + CRYPT_VERIFYCONTEXT)) { + log_notice(LD_CRYPTO, "Unable to set Windows CryptoAPI provider [1]."); + return -1; + } + provider_set = 1; + } + if (!CryptGenRandom(provider, out_len, out)) { + log_notice(LD_CRYPTO, "Unable get entropy from the Windows CryptoAPI."); + return -1; + } + + return 0; +#elif defined(__linux__) && defined(SYS_getrandom) + static int getrandom_works = 1; /* Be optimistic about our chances... */ + + /* getrandom() isn't as straightforward as getentropy(), and has + * no glibc wrapper. + * + * As far as I can tell from getrandom(2) and the source code, the + * requests we issue will always succeed (though it will block on the + * call if /dev/urandom isn't seeded yet), since we are NOT specifying + * GRND_NONBLOCK and the request is <= 256 bytes. + * + * The manpage is unclear on what happens if a signal interrupts the call + * while the request is blocked due to lack of entropy.... + * + * We optimistically assume that getrandom() is available and functional + * because it is the way of the future, and 2 branch mispredicts pale in + * comparison to the overheads involved with failing to open + * /dev/srandom followed by opening and reading from /dev/urandom. + */ + if (PREDICT_LIKELY(getrandom_works)) { + long ret; + /* A flag of '0' here means to read from '/dev/urandom', and to + * block if insufficient entropy is available to service the + * request. + */ + const unsigned int flags = 0; + do { + ret = syscall(SYS_getrandom, out, out_len, flags); + } while (ret == -1 && ((errno == EINTR) ||(errno == EAGAIN))); + + if (PREDICT_UNLIKELY(ret == -1)) { + /* LCOV_EXCL_START we can't actually make the syscall fail in testing. */ + tor_assert(errno != EAGAIN); + tor_assert(errno != EINTR); + + /* Useful log message for errno. */ + if (errno == ENOSYS) { + log_notice(LD_CRYPTO, "Can't get entropy from getrandom()." + " You are running a version of Tor built to support" + " getrandom(), but the kernel doesn't implement this" + " function--probably because it is too old?" + " Trying fallback method instead."); + } else { + log_notice(LD_CRYPTO, "Can't get entropy from getrandom(): %s." + " Trying fallback method instead.", + strerror(errno)); + } + + getrandom_works = 0; /* Don't bother trying again. */ + return -1; + /* LCOV_EXCL_STOP */ + } + + tor_assert(ret == (long)out_len); + return 0; + } + + return -1; /* getrandom() previously failed unexpectedly. */ +#elif defined(HAVE_GETENTROPY) + /* getentropy() is what Linux's getrandom() wants to be when it grows up. + * the only gotcha is that requests are limited to 256 bytes. + */ + return getentropy(out, out_len); +#else + (void) out; +#endif /* defined(_WIN32) || ... */ + + /* This platform doesn't have a supported syscall based random. */ + return -1; +} + +/** + * Try to get <b>out_len</b> bytes of the strongest entropy we can generate, + * via the per-platform fallback mechanism, storing it into <b>out</b>. + * Return 0 on success, -1 on failure. A maximum request size of 256 bytes + * is imposed. + **/ +static int +crypto_strongest_rand_fallback(uint8_t *out, size_t out_len) +{ +#ifdef TOR_UNIT_TESTS + if (break_strongest_rng_fallback) + return -1; +#endif + +#ifdef _WIN32 + /* Windows exclusively uses crypto_strongest_rand_syscall(). */ + (void)out; + (void)out_len; + return -1; +#else /* !(defined(_WIN32)) */ + static const char *filenames[] = { + "/dev/srandom", "/dev/urandom", "/dev/random", NULL + }; + int fd, i; + size_t n; + + for (i = 0; filenames[i]; ++i) { + log_debug(LD_FS, "Considering %s as entropy source", filenames[i]); + fd = open(sandbox_intern_string(filenames[i]), O_RDONLY, 0); + if (fd<0) continue; + log_info(LD_CRYPTO, "Reading entropy from \"%s\"", filenames[i]); + n = read_all(fd, (char*)out, out_len, 0); + close(fd); + if (n != out_len) { + /* LCOV_EXCL_START + * We can't make /dev/foorandom actually fail. */ + log_notice(LD_CRYPTO, + "Error reading from entropy source %s (read only %lu bytes).", + filenames[i], + (unsigned long)n); + return -1; + /* LCOV_EXCL_STOP */ + } + + return 0; + } + + return -1; +#endif /* defined(_WIN32) */ +} + +/** + * Try to get <b>out_len</b> bytes of the strongest entropy we can generate, + * storing it into <b>out</b>. Return 0 on success, -1 on failure. A maximum + * request size of 256 bytes is imposed. + **/ +STATIC int +crypto_strongest_rand_raw(uint8_t *out, size_t out_len) +{ + static const size_t sanity_min_size = 16; + static const int max_attempts = 3; + tor_assert(out_len <= MAX_STRONGEST_RAND_SIZE); + + /* For buffers >= 16 bytes (128 bits), we sanity check the output by + * zero filling the buffer and ensuring that it actually was at least + * partially modified. + * + * Checking that any individual byte is non-zero seems like it would + * fail too often (p = out_len * 1/256) for comfort, but this is an + * "adjust according to taste" sort of check. + */ + memwipe(out, 0, out_len); + for (int i = 0; i < max_attempts; i++) { + /* Try to use the syscall/OS favored mechanism to get strong entropy. */ + if (crypto_strongest_rand_syscall(out, out_len) != 0) { + /* Try to use the less-favored mechanism to get strong entropy. */ + if (crypto_strongest_rand_fallback(out, out_len) != 0) { + /* Welp, we tried. Hopefully the calling code terminates the process + * since we're basically boned without good entropy. + */ + log_warn(LD_CRYPTO, + "Cannot get strong entropy: no entropy source found."); + return -1; + } + } + + if ((out_len < sanity_min_size) || !tor_mem_is_zero((char*)out, out_len)) + return 0; + } + + /* LCOV_EXCL_START + * + * We tried max_attempts times to fill a buffer >= 128 bits long, + * and each time it returned all '0's. Either the system entropy + * source is busted, or the user should go out and buy a ticket to + * every lottery on the planet. + */ + log_warn(LD_CRYPTO, "Strong OS entropy returned all zero buffer."); + + return -1; + /* LCOV_EXCL_STOP */ +} + +/** + * Try to get <b>out_len</b> bytes of the strongest entropy we can generate, + * storing it into <b>out</b>. + **/ +void +crypto_strongest_rand(uint8_t *out, size_t out_len) +{ +#define DLEN SHA512_DIGEST_LENGTH + /* We're going to hash DLEN bytes from the system RNG together with some + * bytes from the openssl PRNG, in order to yield DLEN bytes. + */ + uint8_t inp[DLEN*2]; + uint8_t tmp[DLEN]; + tor_assert(out); + while (out_len) { + crypto_rand((char*) inp, DLEN); + if (crypto_strongest_rand_raw(inp+DLEN, DLEN) < 0) { + // LCOV_EXCL_START + log_err(LD_CRYPTO, "Failed to load strong entropy when generating an " + "important key. Exiting."); + /* Die with an assertion so we get a stack trace. */ + tor_assert(0); + // LCOV_EXCL_STOP + } + if (out_len >= DLEN) { + SHA512(inp, sizeof(inp), out); + out += DLEN; + out_len -= DLEN; + } else { + SHA512(inp, sizeof(inp), tmp); + memcpy(out, tmp, out_len); + break; + } + } + memwipe(tmp, 0, sizeof(tmp)); + memwipe(inp, 0, sizeof(inp)); +#undef DLEN +} + +/** + * Seed OpenSSL's random number generator with bytes from the operating + * system. Return 0 on success, -1 on failure. + **/ +int +crypto_seed_rng(void) +{ + int rand_poll_ok = 0, load_entropy_ok = 0; + uint8_t buf[ADD_ENTROPY]; + + /* OpenSSL has a RAND_poll function that knows about more kinds of + * entropy than we do. We'll try calling that, *and* calling our own entropy + * functions. If one succeeds, we'll accept the RNG as seeded. */ + rand_poll_ok = RAND_poll(); + if (rand_poll_ok == 0) + log_warn(LD_CRYPTO, "RAND_poll() failed."); // LCOV_EXCL_LINE + + load_entropy_ok = !crypto_strongest_rand_raw(buf, sizeof(buf)); + if (load_entropy_ok) { + RAND_seed(buf, sizeof(buf)); + } + + memwipe(buf, 0, sizeof(buf)); + + if ((rand_poll_ok || load_entropy_ok) && RAND_status() == 1) + return 0; + else + return -1; +} + +/** + * Write <b>n</b> bytes of strong random data to <b>to</b>. Supports mocking + * for unit tests. + * + * This function is not allowed to fail; if it would fail to generate strong + * entropy, it must terminate the process instead. + **/ +MOCK_IMPL(void, +crypto_rand, (char *to, size_t n)) +{ + crypto_rand_unmocked(to, n); +} + +/** + * Write <b>n</b> bytes of strong random data to <b>to</b>. Most callers + * will want crypto_rand instead. + * + * This function is not allowed to fail; if it would fail to generate strong + * entropy, it must terminate the process instead. + **/ +void +crypto_rand_unmocked(char *to, size_t n) +{ + int r; + if (n == 0) + return; + + tor_assert(n < INT_MAX); + tor_assert(to); + r = RAND_bytes((unsigned char*)to, (int)n); + /* We consider a PRNG failure non-survivable. Let's assert so that we get a + * stack trace about where it happened. + */ + tor_assert(r >= 0); +} + +/** + * Return a pseudorandom integer, chosen uniformly from the values + * between 0 and <b>max</b>-1 inclusive. <b>max</b> must be between 1 and + * INT_MAX+1, inclusive. + */ +int +crypto_rand_int(unsigned int max) +{ + unsigned int val; + unsigned int cutoff; + tor_assert(max <= ((unsigned int)INT_MAX)+1); + tor_assert(max > 0); /* don't div by 0 */ + + /* We ignore any values that are >= 'cutoff,' to avoid biasing the + * distribution with clipping at the upper end of unsigned int's + * range. + */ + cutoff = UINT_MAX - (UINT_MAX%max); + while (1) { + crypto_rand((char*)&val, sizeof(val)); + if (val < cutoff) + return val % max; + } +} + +/** + * Return a pseudorandom integer, chosen uniformly from the values i such + * that min <= i < max. + * + * <b>min</b> MUST be in range [0, <b>max</b>). + * <b>max</b> MUST be in range (min, INT_MAX]. + **/ +int +crypto_rand_int_range(unsigned int min, unsigned int max) +{ + tor_assert(min < max); + tor_assert(max <= INT_MAX); + + /* The overflow is avoided here because crypto_rand_int() returns a value + * between 0 and (max - min) inclusive. */ + return min + crypto_rand_int(max - min); +} + +/** + * As crypto_rand_int_range, but supports uint64_t. + **/ +uint64_t +crypto_rand_uint64_range(uint64_t min, uint64_t max) +{ + tor_assert(min < max); + return min + crypto_rand_uint64(max - min); +} + +/** + * As crypto_rand_int_range, but supports time_t. + **/ +time_t +crypto_rand_time_range(time_t min, time_t max) +{ + tor_assert(min < max); + return min + (time_t)crypto_rand_uint64(max - min); +} + +/** + * Return a pseudorandom 64-bit integer, chosen uniformly from the values + * between 0 and <b>max</b>-1 inclusive. + **/ +uint64_t +crypto_rand_uint64(uint64_t max) +{ + uint64_t val; + uint64_t cutoff; + tor_assert(max < UINT64_MAX); + tor_assert(max > 0); /* don't div by 0 */ + + /* We ignore any values that are >= 'cutoff,' to avoid biasing the + * distribution with clipping at the upper end of unsigned int's + * range. + */ + cutoff = UINT64_MAX - (UINT64_MAX%max); + while (1) { + crypto_rand((char*)&val, sizeof(val)); + if (val < cutoff) + return val % max; + } +} + +/** + * Return a pseudorandom double d, chosen uniformly from the range + * 0.0 <= d < 1.0. + **/ +double +crypto_rand_double(void) +{ + /* We just use an unsigned int here; we don't really care about getting + * more than 32 bits of resolution */ + unsigned int u; + crypto_rand((char*)&u, sizeof(u)); +#if SIZEOF_INT == 4 +#define UINT_MAX_AS_DOUBLE 4294967296.0 +#elif SIZEOF_INT == 8 +#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19 +#else +#error SIZEOF_INT is neither 4 nor 8 +#endif /* SIZEOF_INT == 4 || ... */ + return ((double)u) / UINT_MAX_AS_DOUBLE; +} + +/** + * Generate and return a new random hostname starting with <b>prefix</b>, + * ending with <b>suffix</b>, and containing no fewer than + * <b>min_rand_len</b> and no more than <b>max_rand_len</b> random base32 + * characters. Does not check for failure. + * + * Clip <b>max_rand_len</b> to MAX_DNS_LABEL_SIZE. + **/ +char * +crypto_random_hostname(int min_rand_len, int max_rand_len, const char *prefix, + const char *suffix) +{ + char *result, *rand_bytes; + int randlen, rand_bytes_len; + size_t resultlen, prefixlen; + + if (max_rand_len > MAX_DNS_LABEL_SIZE) + max_rand_len = MAX_DNS_LABEL_SIZE; + if (min_rand_len > max_rand_len) + min_rand_len = max_rand_len; + + randlen = crypto_rand_int_range(min_rand_len, max_rand_len+1); + + prefixlen = strlen(prefix); + resultlen = prefixlen + strlen(suffix) + randlen + 16; + + rand_bytes_len = ((randlen*5)+7)/8; + if (rand_bytes_len % 5) + rand_bytes_len += 5 - (rand_bytes_len%5); + rand_bytes = tor_malloc(rand_bytes_len); + crypto_rand(rand_bytes, rand_bytes_len); + + result = tor_malloc(resultlen); + memcpy(result, prefix, prefixlen); + base32_encode(result+prefixlen, resultlen-prefixlen, + rand_bytes, rand_bytes_len); + tor_free(rand_bytes); + strlcpy(result+prefixlen+randlen, suffix, resultlen-(prefixlen+randlen)); + + return result; +} + +/** + * Return a randomly chosen element of <b>sl</b>; or NULL if <b>sl</b> + * is empty. + **/ +void * +smartlist_choose(const smartlist_t *sl) +{ + int len = smartlist_len(sl); + if (len) + return smartlist_get(sl,crypto_rand_int(len)); + return NULL; /* no elements to choose from */ +} + +/** + * Scramble the elements of <b>sl</b> into a random order. + **/ +void +smartlist_shuffle(smartlist_t *sl) +{ + int i; + /* From the end of the list to the front, choose at random from the + positions we haven't looked at yet, and swap that position into the + current position. Remember to give "no swap" the same probability as + any other swap. */ + for (i = smartlist_len(sl)-1; i > 0; --i) { + int j = crypto_rand_int(i+1); + smartlist_swap(sl, i, j); + } +} + +/** Make sure that openssl is using its default PRNG. Return 1 if we had to + * adjust it; 0 otherwise. */ +int +crypto_force_rand_ssleay(void) +{ + RAND_METHOD *default_method; + default_method = RAND_OpenSSL(); + if (RAND_get_rand_method() != default_method) { + log_notice(LD_CRYPTO, "It appears that one of our engines has provided " + "a replacement the OpenSSL RNG. Resetting it to the default " + "implementation."); + RAND_set_rand_method(default_method); + return 1; + } + return 0; +} + +#endif /* !defined(CRYPTO_RAND_PRIVATE) */ + diff --git a/src/common/crypto_rand.h b/src/common/crypto_rand.h new file mode 100644 index 0000000000..bb02e51001 --- /dev/null +++ b/src/common/crypto_rand.h @@ -0,0 +1,52 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_rand.h + * + * \brief Common functions for using (pseudo-)random number generators. + **/ + +#ifndef TOR_CRYPTO_RAND_H +#define TOR_CRYPTO_RAND_H + +#include "torint.h" +#include "util.h" + +/* random numbers */ +int crypto_seed_rng(void) ATTR_WUR; +MOCK_DECL(void,crypto_rand,(char *to, size_t n)); +void crypto_rand_unmocked(char *to, size_t n); +void crypto_strongest_rand(uint8_t *out, size_t out_len); +int crypto_rand_int(unsigned int max); +int crypto_rand_int_range(unsigned int min, unsigned int max); +uint64_t crypto_rand_uint64_range(uint64_t min, uint64_t max); +time_t crypto_rand_time_range(time_t min, time_t max); +uint64_t crypto_rand_uint64(uint64_t max); +double crypto_rand_double(void); +struct tor_weak_rng_t; +void crypto_seed_weak_rng(struct tor_weak_rng_t *rng); + +char *crypto_random_hostname(int min_rand_len, int max_rand_len, + const char *prefix, const char *suffix); + +struct smartlist_t; +void *smartlist_choose(const struct smartlist_t *sl); +void smartlist_shuffle(struct smartlist_t *sl); +int crypto_force_rand_ssleay(void); + +#ifdef CRYPTO_RAND_PRIVATE + +STATIC int crypto_strongest_rand_raw(uint8_t *out, size_t out_len); + +#ifdef TOR_UNIT_TESTS +extern int break_strongest_rng_syscall; +extern int break_strongest_rng_fallback; +#endif +#endif /* defined(CRYPTO_RAND_PRIVATE) */ + +#endif /* !defined(TOR_CRYPTO_RAND_H) */ + diff --git a/src/common/crypto_rsa.c b/src/common/crypto_rsa.c index 259656810b..f66cdef3c5 100644 --- a/src/common/crypto_rsa.c +++ b/src/common/crypto_rsa.c @@ -9,12 +9,14 @@ * \brief Block of functions related with RSA utilities and operations. **/ -#include "crypto_rsa.h" #include "crypto.h" -#include "compat_openssl.h" #include "crypto_curve25519.h" -#include "crypto_ed25519.h" +#include "crypto_digest.h" #include "crypto_format.h" +#include "compat_openssl.h" +#include "crypto_rand.h" +#include "crypto_rsa.h" +#include "crypto_util.h" DISABLE_GCC_WARNING(redundant-decls) @@ -627,6 +629,148 @@ crypto_pk_copy_full(crypto_pk_t *env) return crypto_new_pk_from_rsa_(new_key); } +/** Perform a hybrid (public/secret) encryption on <b>fromlen</b> + * bytes of data from <b>from</b>, with padding type 'padding', + * storing the results on <b>to</b>. + * + * Returns the number of bytes written on success, -1 on failure. + * + * The encrypted data consists of: + * - The source data, padded and encrypted with the public key, if the + * padded source data is no longer than the public key, and <b>force</b> + * is false, OR + * - The beginning of the source data prefixed with a 16-byte symmetric key, + * padded and encrypted with the public key; followed by the rest of + * the source data encrypted in AES-CTR mode with the symmetric key. + * + * NOTE that this format does not authenticate the symmetrically encrypted + * part of the data, and SHOULD NOT BE USED for new protocols. + */ +int +crypto_pk_obsolete_public_hybrid_encrypt(crypto_pk_t *env, + char *to, size_t tolen, + const char *from, + size_t fromlen, + int padding, int force) +{ + int overhead, outlen, r; + size_t pkeylen, symlen; + crypto_cipher_t *cipher = NULL; + char *buf = NULL; + + tor_assert(env); + tor_assert(from); + tor_assert(to); + tor_assert(fromlen < SIZE_T_CEILING); + + overhead = crypto_get_rsa_padding_overhead(crypto_get_rsa_padding(padding)); + pkeylen = crypto_pk_keysize(env); + + if (!force && fromlen+overhead <= pkeylen) { + /* It all fits in a single encrypt. */ + return crypto_pk_public_encrypt(env,to, + tolen, + from,fromlen,padding); + } + tor_assert(tolen >= fromlen + overhead + CIPHER_KEY_LEN); + tor_assert(tolen >= pkeylen); + + char key[CIPHER_KEY_LEN]; + crypto_rand(key, sizeof(key)); /* generate a new key. */ + cipher = crypto_cipher_new(key); + + buf = tor_malloc(pkeylen+1); + memcpy(buf, key, CIPHER_KEY_LEN); + memcpy(buf+CIPHER_KEY_LEN, from, pkeylen-overhead-CIPHER_KEY_LEN); + + /* Length of symmetrically encrypted data. */ + symlen = fromlen-(pkeylen-overhead-CIPHER_KEY_LEN); + + outlen = crypto_pk_public_encrypt(env,to,tolen,buf,pkeylen-overhead,padding); + if (outlen!=(int)pkeylen) { + goto err; + } + r = crypto_cipher_encrypt(cipher, to+outlen, + from+pkeylen-overhead-CIPHER_KEY_LEN, symlen); + + if (r<0) goto err; + memwipe(buf, 0, pkeylen); + memwipe(key, 0, sizeof(key)); + tor_free(buf); + crypto_cipher_free(cipher); + tor_assert(outlen+symlen < INT_MAX); + return (int)(outlen + symlen); + err: + + memwipe(buf, 0, pkeylen); + memwipe(key, 0, sizeof(key)); + tor_free(buf); + crypto_cipher_free(cipher); + return -1; +} + +/** Invert crypto_pk_obsolete_public_hybrid_encrypt. Returns the number of + * bytes written on success, -1 on failure. + * + * NOTE that this format does not authenticate the symmetrically encrypted + * part of the data, and SHOULD NOT BE USED for new protocols. + */ +int +crypto_pk_obsolete_private_hybrid_decrypt(crypto_pk_t *env, + char *to, + size_t tolen, + const char *from, + size_t fromlen, + int padding, int warnOnFailure) +{ + int outlen, r; + size_t pkeylen; + crypto_cipher_t *cipher = NULL; + char *buf = NULL; + + tor_assert(fromlen < SIZE_T_CEILING); + pkeylen = crypto_pk_keysize(env); + + if (fromlen <= pkeylen) { + return crypto_pk_private_decrypt(env,to,tolen,from,fromlen,padding, + warnOnFailure); + } + + buf = tor_malloc(pkeylen); + outlen = crypto_pk_private_decrypt(env,buf,pkeylen,from,pkeylen,padding, + warnOnFailure); + if (outlen<0) { + log_fn(warnOnFailure?LOG_WARN:LOG_DEBUG, LD_CRYPTO, + "Error decrypting public-key data"); + goto err; + } + if (outlen < CIPHER_KEY_LEN) { + log_fn(warnOnFailure?LOG_WARN:LOG_INFO, LD_CRYPTO, + "No room for a symmetric key"); + goto err; + } + cipher = crypto_cipher_new(buf); + if (!cipher) { + goto err; + } + memcpy(to,buf+CIPHER_KEY_LEN,outlen-CIPHER_KEY_LEN); + outlen -= CIPHER_KEY_LEN; + tor_assert(tolen - outlen >= fromlen - pkeylen); + r = crypto_cipher_decrypt(cipher, to+outlen, from+pkeylen, fromlen-pkeylen); + if (r<0) + goto err; + memwipe(buf,0,pkeylen); + tor_free(buf); + crypto_cipher_free(cipher); + tor_assert(outlen + fromlen < INT_MAX); + return (int)(outlen + (fromlen-pkeylen)); + err: + memwipe(buf,0,pkeylen); + tor_free(buf); + crypto_cipher_free(cipher); + return -1; +} + /** Encrypt <b>fromlen</b> bytes from <b>from</b> with the public key * in <b>env</b>, using the padding method <b>padding</b>. On success, * write the result to <b>to</b>, and return the number of bytes @@ -849,6 +993,122 @@ crypto_pk_get_hashed_fingerprint(crypto_pk_t *pk, char *fp_out) return 0; } +/** Check a siglen-byte long signature at <b>sig</b> against + * <b>datalen</b> bytes of data at <b>data</b>, using the public key + * in <b>env</b>. Return 0 if <b>sig</b> is a correct signature for + * SHA1(data). Else return -1. + */ +MOCK_IMPL(int, +crypto_pk_public_checksig_digest,(crypto_pk_t *env, const char *data, + size_t datalen, const char *sig, + size_t siglen)) +{ + char digest[DIGEST_LEN]; + char *buf; + size_t buflen; + int r; + + tor_assert(env); + tor_assert(data); + tor_assert(sig); + tor_assert(datalen < SIZE_T_CEILING); + tor_assert(siglen < SIZE_T_CEILING); + + if (crypto_digest(digest,data,datalen)<0) { + log_warn(LD_BUG, "couldn't compute digest"); + return -1; + } + buflen = crypto_pk_keysize(env); + buf = tor_malloc(buflen); + r = crypto_pk_public_checksig(env,buf,buflen,sig,siglen); + if (r != DIGEST_LEN) { + log_warn(LD_CRYPTO, "Invalid signature"); + tor_free(buf); + return -1; + } + if (tor_memneq(buf, digest, DIGEST_LEN)) { + log_warn(LD_CRYPTO, "Signature mismatched with digest."); + tor_free(buf); + return -1; + } + tor_free(buf); + + return 0; +} + +/** Compute a SHA1 digest of <b>fromlen</b> bytes of data stored at + * <b>from</b>; sign the data with the private key in <b>env</b>, and + * store it in <b>to</b>. Return the number of bytes written on + * success, and -1 on failure. + * + * <b>tolen</b> is the number of writable bytes in <b>to</b>, and must be + * at least the length of the modulus of <b>env</b>. + */ +int +crypto_pk_private_sign_digest(crypto_pk_t *env, char *to, size_t tolen, + const char *from, size_t fromlen) +{ + int r; + char digest[DIGEST_LEN]; + if (crypto_digest(digest,from,fromlen)<0) + return -1; + r = crypto_pk_private_sign(env,to,tolen,digest,DIGEST_LEN); + memwipe(digest, 0, sizeof(digest)); + return r; +} + +/** Given a private or public key <b>pk</b>, put a SHA1 hash of the + * public key into <b>digest_out</b> (must have DIGEST_LEN bytes of space). + * Return 0 on success, -1 on failure. + */ +int +crypto_pk_get_digest(const crypto_pk_t *pk, char *digest_out) +{ + char *buf; + size_t buflen; + int len; + int rv = -1; + + buflen = crypto_pk_keysize(pk)*2; + buf = tor_malloc(buflen); + len = crypto_pk_asn1_encode(pk, buf, buflen); + if (len < 0) + goto done; + + if (crypto_digest(digest_out, buf, len) < 0) + goto done; + + rv = 0; + done: + tor_free(buf); + return rv; +} + +/** Compute all digests of the DER encoding of <b>pk</b>, and store them + * in <b>digests_out</b>. Return 0 on success, -1 on failure. */ +int +crypto_pk_get_common_digests(crypto_pk_t *pk, common_digests_t *digests_out) +{ + char *buf; + size_t buflen; + int len; + int rv = -1; + + buflen = crypto_pk_keysize(pk)*2; + buf = tor_malloc(buflen); + len = crypto_pk_asn1_encode(pk, buf, buflen); + if (len < 0) + goto done; + + if (crypto_common_digests(digests_out, (char*)buf, len) < 0) + goto done; + + rv = 0; + done: + tor_free(buf); + return rv; +} + /** Given a crypto_pk_t <b>pk</b>, allocate a new buffer containing the * Base64 encoding of the DER representation of the private key as a NUL * terminated string, and return it via <b>priv_out</b>. Return 0 on diff --git a/src/common/crypto_rsa.h b/src/common/crypto_rsa.h index 5b9025c629..e952089318 100644 --- a/src/common/crypto_rsa.h +++ b/src/common/crypto_rsa.h @@ -15,13 +15,13 @@ #include "orconfig.h" +#include "crypto_digest.h" #include <stdio.h> #include "torint.h" #include "testsupport.h" #include "compat.h" #include "util.h" #include "torlog.h" -#include "crypto_curve25519.h" /** Length of our public keys. */ #define PK_BYTES (1024/8) @@ -35,7 +35,7 @@ /** A public key, or a public/private key-pair. */ typedef struct crypto_pk_t crypto_pk_t; -/* RSA enviroment setup */ +/* RSA environment setup */ MOCK_DECL(crypto_pk_t *,crypto_pk_new,(void)); void crypto_pk_free_(crypto_pk_t *env); #define crypto_pk_free(pk) FREE_AND_NULL(crypto_pk_t, crypto_pk_free_, (pk)) @@ -69,6 +69,14 @@ crypto_pk_t *crypto_pk_dup_key(crypto_pk_t *orig); crypto_pk_t *crypto_pk_copy_full(crypto_pk_t *orig); int crypto_pk_key_is_private(const crypto_pk_t *key); int crypto_pk_public_exponent_ok(crypto_pk_t *env); +int crypto_pk_obsolete_public_hybrid_encrypt(crypto_pk_t *env, char *to, + size_t tolen, + const char *from, size_t fromlen, + int padding, int force); +int crypto_pk_obsolete_private_hybrid_decrypt(crypto_pk_t *env, char *to, + size_t tolen, + const char *from, size_t fromlen, + int padding, int warnOnFailure); int crypto_pk_public_encrypt(crypto_pk_t *env, char *to, size_t tolen, const char *from, size_t fromlen, int padding); int crypto_pk_private_decrypt(crypto_pk_t *env, char *to, size_t tolen, @@ -84,6 +92,13 @@ crypto_pk_t *crypto_pk_asn1_decode(const char *str, size_t len); int crypto_pk_get_fingerprint(crypto_pk_t *pk, char *fp_out,int add_space); int crypto_pk_get_hashed_fingerprint(crypto_pk_t *pk, char *fp_out); +MOCK_DECL(int, crypto_pk_public_checksig_digest,(crypto_pk_t *env, + const char *data, size_t datalen, const char *sig, size_t siglen)); +int crypto_pk_private_sign_digest(crypto_pk_t *env, char *to, size_t tolen, + const char *from, size_t fromlen); +int crypto_pk_get_digest(const crypto_pk_t *pk, char *digest_out); +int crypto_pk_get_common_digests(crypto_pk_t *pk, + common_digests_t *digests_out); int crypto_pk_base64_encode(const crypto_pk_t *pk, char **priv_out); crypto_pk_t *crypto_pk_base64_decode(const char *str, size_t len); diff --git a/src/common/crypto_s2k.c b/src/common/crypto_s2k.c index b2fcca54c4..8543760ec5 100644 --- a/src/common/crypto_s2k.c +++ b/src/common/crypto_s2k.c @@ -12,10 +12,13 @@ #define CRYPTO_S2K_PRIVATE -#include "crypto.h" -#include "util.h" #include "compat.h" +#include "crypto.h" +#include "crypto_digest.h" +#include "crypto_rand.h" #include "crypto_s2k.h" +#include "crypto_util.h" +#include "util.h" #include <openssl/evp.h> diff --git a/src/common/crypto_util.c b/src/common/crypto_util.c new file mode 100644 index 0000000000..b0d5b6b2f7 --- /dev/null +++ b/src/common/crypto_util.c @@ -0,0 +1,107 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_util.c + * + * \brief Common cryptographic utilities. + **/ + +#ifndef CRYPTO_UTIL_PRIVATE +#define CRYPTO_UTIL_PRIVATE + +#include "crypto_util.h" + +#include <string.h> + +#ifdef _WIN32 +#include <winsock2.h> +#include <windows.h> +#include <wincrypt.h> +#endif /* defined(_WIN32) */ + +#include "util.h" + +DISABLE_GCC_WARNING(redundant-decls) + +#include <openssl/crypto.h> + +ENABLE_GCC_WARNING(redundant-decls) + +/** + * Destroy the <b>sz</b> bytes of data stored at <b>mem</b>, setting them to + * the value <b>byte</b>. + * If <b>mem</b> is NULL or <b>sz</b> is zero, nothing happens. + * + * This function is preferable to memset, since many compilers will happily + * optimize out memset() when they can convince themselves that the data being + * cleared will never be read. + * + * Right now, our convention is to use this function when we are wiping data + * that's about to become inaccessible, such as stack buffers that are about + * to go out of scope or structures that are about to get freed. (In + * practice, it appears that the compilers we're currently using will optimize + * out the memset()s for stack-allocated buffers, but not those for + * about-to-be-freed structures. That could change, though, so we're being + * wary.) If there are live reads for the data, then you can just use + * memset(). + */ +void +memwipe(void *mem, uint8_t byte, size_t sz) +{ + if (sz == 0) { + return; + } + /* If sz is nonzero, then mem must not be NULL. */ + tor_assert(mem != NULL); + + /* Data this large is likely to be an underflow. */ + tor_assert(sz < SIZE_T_CEILING); + + /* Because whole-program-optimization exists, we may not be able to just + * have this function call "memset". A smart compiler could inline it, then + * eliminate dead memsets, and declare itself to be clever. */ + +#if defined(SecureZeroMemory) || defined(HAVE_SECUREZEROMEMORY) + /* Here's what you do on windows. */ + SecureZeroMemory(mem,sz); +#elif defined(HAVE_RTLSECUREZEROMEMORY) + RtlSecureZeroMemory(mem,sz); +#elif defined(HAVE_EXPLICIT_BZERO) + /* The BSDs provide this. */ + explicit_bzero(mem, sz); +#elif defined(HAVE_MEMSET_S) + /* This is in the C99 standard. */ + memset_s(mem, sz, 0, sz); +#else + /* This is a slow and ugly function from OpenSSL that fills 'mem' with junk + * based on the pointer value, then uses that junk to update a global + * variable. It's an elaborate ruse to trick the compiler into not + * optimizing out the "wipe this memory" code. Read it if you like zany + * programming tricks! In later versions of Tor, we should look for better + * not-optimized-out memory wiping stuff... + * + * ...or maybe not. In practice, there are pure-asm implementations of + * OPENSSL_cleanse() on most platforms, which ought to do the job. + **/ + + OPENSSL_cleanse(mem, sz); +#endif /* defined(SecureZeroMemory) || defined(HAVE_SECUREZEROMEMORY) || ... */ + + /* Just in case some caller of memwipe() is relying on getting a buffer + * filled with a particular value, fill the buffer. + * + * If this function gets inlined, this memset might get eliminated, but + * that's okay: We only care about this particular memset in the case where + * the caller should have been using memset(), and the memset() wouldn't get + * eliminated. In other words, this is here so that we won't break anything + * if somebody accidentally calls memwipe() instead of memset(). + **/ + memset(mem, byte, sz); +} + +#endif /* !defined(CRYPTO_UTIL_PRIVATE) */ + diff --git a/src/common/crypto_util.h b/src/common/crypto_util.h new file mode 100644 index 0000000000..922942b371 --- /dev/null +++ b/src/common/crypto_util.h @@ -0,0 +1,27 @@ +/* Copyright (c) 2001, Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file crypto_util.h + * + * \brief Common functions for cryptographic routines. + **/ + +#ifndef TOR_CRYPTO_UTIL_H +#define TOR_CRYPTO_UTIL_H + +#include "torint.h" + +/** OpenSSL-based utility functions. */ +void memwipe(void *mem, uint8_t byte, size_t sz); + +#ifdef CRYPTO_UTIL_PRIVATE +#ifdef TOR_UNIT_TESTS +#endif /* defined(TOR_UNIT_TESTS) */ +#endif /* defined(CRYPTO_UTIL_PRIVATE) */ + +#endif /* !defined(TOR_CRYPTO_UTIL_H) */ + diff --git a/src/common/include.am b/src/common/include.am index 6945285108..cfaf993674 100644 --- a/src/common/include.am +++ b/src/common/include.am @@ -74,7 +74,11 @@ LIBOR_CTIME_A_SRC = \ src/common/di_ops.c src_common_libor_ctime_a_SOURCES = $(LIBOR_CTIME_A_SRC) +if UNITTESTS_ENABLED src_common_libor_ctime_testing_a_SOURCES = $(LIBOR_CTIME_A_SRC) +else +src_common_libor_ctime_testing_a_SOURCES = +endif src_common_libor_ctime_a_CFLAGS = @CFLAGS_CONSTTIME@ src_common_libor_ctime_testing_a_CFLAGS = @CFLAGS_CONSTTIME@ $(TEST_CFLAGS) @@ -97,6 +101,7 @@ LIBOR_A_SRC = \ src/common/util_process.c \ src/common/sandbox.c \ src/common/storagedir.c \ + src/common/token_bucket.c \ src/common/workqueue.c \ $(libor_extra_source) \ $(threads_impl_source) \ @@ -114,11 +119,14 @@ LIBOR_CRYPTO_A_SRC = \ src/common/compress_zlib.c \ src/common/compress_zstd.c \ src/common/crypto.c \ - src/common/crypto_rsa.c \ + src/common/crypto_digest.c \ + src/common/crypto_format.c \ src/common/crypto_openssl_mgt.c \ src/common/crypto_pwbox.c \ + src/common/crypto_rand.c \ + src/common/crypto_rsa.c \ src/common/crypto_s2k.c \ - src/common/crypto_format.c \ + src/common/crypto_util.c \ src/common/tortls.c \ src/common/crypto_curve25519.c \ src/common/crypto_ed25519.c @@ -133,9 +141,15 @@ src_common_libor_a_SOURCES = $(LIBOR_A_SRC) src_common_libor_crypto_a_SOURCES = $(LIBOR_CRYPTO_A_SRC) src_common_libor_event_a_SOURCES = $(LIBOR_EVENT_A_SRC) +if UNITTESTS_ENABLED src_common_libor_testing_a_SOURCES = $(LIBOR_A_SRC) src_common_libor_crypto_testing_a_SOURCES = $(LIBOR_CRYPTO_A_SRC) src_common_libor_event_testing_a_SOURCES = $(LIBOR_EVENT_A_SRC) +else +src_common_libor_testing_a_SOURCES = +src_common_libor_crypto_testing_a_SOURCES = +src_common_libor_event_testing_a_SOURCES = +endif src_common_libor_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_common_libor_crypto_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS) @@ -165,13 +179,16 @@ COMMONHEADERS = \ src/common/confline.h \ src/common/container.h \ src/common/crypto.h \ + src/common/crypto_digest.h \ src/common/crypto_curve25519.h \ src/common/crypto_ed25519.h \ src/common/crypto_format.h \ src/common/crypto_openssl_mgt.h \ - src/common/crypto_rsa.h \ src/common/crypto_pwbox.h \ + src/common/crypto_rand.h \ + src/common/crypto_rsa.h \ src/common/crypto_s2k.h \ + src/common/crypto_util.h \ src/common/di_ops.h \ src/common/handles.h \ src/common/memarea.h \ @@ -182,6 +199,7 @@ COMMONHEADERS = \ src/common/storagedir.h \ src/common/testsupport.h \ src/common/timers.h \ + src/common/token_bucket.h \ src/common/torint.h \ src/common/torlog.h \ src/common/tortls.h \ diff --git a/src/common/log.c b/src/common/log.c index 9f4a8b2bc2..ebd50f62d3 100644 --- a/src/common/log.c +++ b/src/common/log.c @@ -52,6 +52,13 @@ #define raw_assert(x) assert(x) // assert OK +/** Defining compile-time constants for Tor log levels (used by the Rust + * log wrapper at src/rust/tor_log) */ +const int LOG_WARN_ = LOG_WARN; +const int LOG_NOTICE_ = LOG_NOTICE; +const log_domain_mask_t LD_GENERAL_ = LD_GENERAL; +const log_domain_mask_t LD_NET_ = LD_NET; + /** Information for a single logfile; only used in log.c */ typedef struct logfile_t { struct logfile_t *next; /**< Next logfile_t in the linked list. */ @@ -163,6 +170,9 @@ typedef struct pending_log_message_t { /** Log messages waiting to be replayed onto callback-based logs */ static smartlist_t *pending_cb_messages = NULL; +/** Callback to invoke when pending_cb_messages becomes nonempty. */ +static pending_callback_callback pending_cb_cb = NULL; + /** Log messages waiting to be replayed once the logging system is initialized. */ static smartlist_t *pending_startup_messages = NULL; @@ -225,6 +235,30 @@ log_set_application_name(const char *name) appname = name ? tor_strdup(name) : NULL; } +/** Return true if some of the running logs might be interested in a log + * message of the given severity in the given domains. If this function + * returns true, the log message might be ignored anyway, but if it returns + * false, it is definitely_ safe not to log the message. */ +int +log_message_is_interesting(int severity, log_domain_mask_t domain) +{ + (void) domain; + return (severity <= log_global_min_severity_); +} + +/** + * As tor_log, but takes an optional function name, and does not treat its + * <b>string</b> as a printf format. + * + * For use by Rust integration. + */ +void +tor_log_string(int severity, log_domain_mask_t domain, + const char *function, const char *string) +{ + log_fn_(severity, domain, function, "%s", string); +} + /** Log time granularity in milliseconds. */ static int log_time_granularity = 1; @@ -507,6 +541,9 @@ logfile_deliver(logfile_t *lf, const char *buf, size_t msg_len, smartlist_add(pending_cb_messages, pending_log_message_new(severity,domain,NULL,msg_after_prefix)); *callbacks_deferred = 1; + if (smartlist_len(pending_cb_messages) == 1 && pending_cb_cb) { + pending_cb_cb(); + } } } else { lf->callback(severity, domain, msg_after_prefix); @@ -794,6 +831,7 @@ logs_free_all(void) logfiles = NULL; messages = pending_cb_messages; pending_cb_messages = NULL; + pending_cb_cb = NULL; messages2 = pending_startup_messages; pending_startup_messages = NULL; UNLOCK_LOGS(); @@ -957,6 +995,24 @@ add_temp_log(int min_severity) } /** + * Register "cb" as the callback to call when there are new pending log + * callbacks to be flushed with flush_pending_log_callbacks(). + * + * Note that this callback, if present, can be invoked from any thread. + * + * This callback must not log. + * + * It is intentional that this function contains the name "callback" twice: it + * sets a "callback" to be called on the condition that there is a "pending + * callback". + **/ +void +logs_set_pending_callback_callback(pending_callback_callback cb) +{ + pending_cb_cb = cb; +} + +/** * Add a log handler to send messages in <b>severity</b> * to the function <b>cb</b>. */ diff --git a/src/common/procmon.c b/src/common/procmon.c index abcbbeaa21..73c14cd584 100644 --- a/src/common/procmon.c +++ b/src/common/procmon.c @@ -10,8 +10,6 @@ #include "util.h" -#include <event2/event.h> - #ifdef HAVE_SIGNAL_H #include <signal.h> #endif @@ -44,7 +42,7 @@ typedef int pid_t; /* Currently we need to poll in some way on all systems. */ #ifdef PROCMON_POLLS -static void tor_process_monitor_poll_cb(evutil_socket_t unused1, short unused2, +static void tor_process_monitor_poll_cb(periodic_timer_t *ev, void *procmon_); #endif @@ -136,7 +134,7 @@ struct tor_process_monitor_t { /** A Libevent event structure, to either poll for the process's * existence or receive a notification when the process ends. */ - struct event *e; + periodic_timer_t *e; /** A callback to be called when the process ends. */ tor_procmon_callback_t cb; @@ -159,9 +157,6 @@ tor_validate_process_specifier(const char *process_spec, return parse_process_specifier(process_spec, &ppspec, msg); } -/* XXXX we should use periodic_timer_new() for this stuff */ -#define PERIODIC_TIMER_FLAGS EV_PERSIST - /* DOCDOC poll_interval_tv */ static const struct timeval poll_interval_tv = {15, 0}; @@ -225,13 +220,9 @@ tor_process_monitor_new(struct event_base *base, procmon->cb_arg = cb_arg; #ifdef PROCMON_POLLS - procmon->e = tor_event_new(base, -1 /* no FD */, PERIODIC_TIMER_FLAGS, - tor_process_monitor_poll_cb, procmon); - /* Note: If you port this file to plain Libevent 2, check that - * procmon->e is non-NULL. We don't need to here because - * tor_evtimer_new never returns NULL. */ - - evtimer_add(procmon->e, &poll_interval_tv); + procmon->e = periodic_timer_new(base, + &poll_interval_tv, + tor_process_monitor_poll_cb, procmon); #else /* !(defined(PROCMON_POLLS)) */ #error OOPS? #endif /* defined(PROCMON_POLLS) */ @@ -246,14 +237,12 @@ tor_process_monitor_new(struct event_base *base, /** Libevent callback to poll for the existence of the process * monitored by <b>procmon_</b>. */ static void -tor_process_monitor_poll_cb(evutil_socket_t unused1, short unused2, - void *procmon_) +tor_process_monitor_poll_cb(periodic_timer_t *event, void *procmon_) { + (void)event; tor_process_monitor_t *procmon = (tor_process_monitor_t *)(procmon_); int its_dead_jim; - (void)unused1; (void)unused2; - tor_assert(procmon != NULL); #ifdef _WIN32 @@ -336,7 +325,7 @@ tor_process_monitor_free_(tor_process_monitor_t *procmon) #endif if (procmon->e != NULL) - tor_event_free(procmon->e); + periodic_timer_free(procmon->e); tor_free(procmon); } diff --git a/src/common/timers.c b/src/common/timers.c index 552080b11e..6f6236ed3b 100644 --- a/src/common/timers.c +++ b/src/common/timers.c @@ -37,8 +37,6 @@ #include "torlog.h" #include "util.h" -#include <event2/event.h> - struct timeout_cb { timer_cb_fn_t cb; void *arg; @@ -66,10 +64,15 @@ struct timeout_cb { * above TIMEOUT_MAX can also be super-inefficient. Choosing 5 here sets * timeout_max to 2^30 ticks, or 29 hours with our value for USEC_PER_TICK */ #define WHEEL_NUM 5 +#if SIZEOF_VOID_P == 4 +/* On 32-bit platforms, we want to override wheel_bit, so that timeout.c will + * use 32-bit math. */ +#define WHEEL_BIT 5 +#endif #include "src/ext/timeouts/timeout.c" static struct timeouts *global_timeouts = NULL; -static struct event *global_timer_event = NULL; +static struct mainloop_event_t *global_timer_event = NULL; static monotime_t start_of_time; @@ -147,7 +150,7 @@ libevent_timer_reschedule(void) if (delay > MIN_CHECK_TICKS) delay = MIN_CHECK_TICKS; timeout_to_tv(delay, &d); - event_add(global_timer_event, &d); + mainloop_event_schedule(global_timer_event, &d); } /** Run the callback of every timer that has expired, based on the current @@ -170,10 +173,9 @@ timers_run_pending(void) * have fired, activate their callbacks, and reschedule the libevent timer. */ static void -libevent_timer_callback(evutil_socket_t fd, short what, void *arg) +libevent_timer_callback(mainloop_event_t *ev, void *arg) { - (void)fd; - (void)what; + (void)ev; (void)arg; timers_run_pending(); @@ -203,9 +205,8 @@ timers_initialize(void) monotime_init(); monotime_get(&start_of_time); - struct event *timer_event; - timer_event = tor_event_new(tor_libevent_get_base(), - -1, 0, libevent_timer_callback, NULL); + mainloop_event_t *timer_event; + timer_event = mainloop_event_new(libevent_timer_callback, NULL); tor_assert(timer_event); global_timer_event = timer_event; @@ -219,7 +220,7 @@ void timers_shutdown(void) { if (global_timer_event) { - tor_event_free(global_timer_event); + mainloop_event_free(global_timer_event); global_timer_event = NULL; } if (global_timeouts) { diff --git a/src/common/token_bucket.c b/src/common/token_bucket.c new file mode 100644 index 0000000000..f2396ec58a --- /dev/null +++ b/src/common/token_bucket.c @@ -0,0 +1,255 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file token_bucket.c + * \brief Functions to use and manipulate token buckets, used for + * rate-limiting on connections and globally. + * + * Tor uses these token buckets to keep track of bandwidth usage, and + * sometimes other things too. + * + * There are two layers of abstraction here: "raw" token buckets, in which all + * the pieces are decoupled, and "read-write" token buckets, which combine all + * the moving parts into one. + * + * Token buckets may become negative. + **/ + +#define TOKEN_BUCKET_PRIVATE + +#include "token_bucket.h" +#include "util_bug.h" + +/** + * Set the <b>rate</b> and <b>burst</b> value in a token_bucket_cfg. + * + * Note that the <b>rate</b> value is in arbitrary units, but those units will + * determine the units of token_bucket_raw_dec(), token_bucket_raw_refill, and + * so on. + */ +void +token_bucket_cfg_init(token_bucket_cfg_t *cfg, + uint32_t rate, + uint32_t burst) +{ + tor_assert_nonfatal(rate > 0); + tor_assert_nonfatal(burst > 0); + if (burst > TOKEN_BUCKET_MAX_BURST) + burst = TOKEN_BUCKET_MAX_BURST; + + cfg->rate = rate; + cfg->burst = burst; +} + +/** + * Initialize a raw token bucket and its associated timestamp to the "full" + * state, according to <b>cfg</b>. + */ +void +token_bucket_raw_reset(token_bucket_raw_t *bucket, + const token_bucket_cfg_t *cfg) +{ + bucket->bucket = cfg->burst; +} + +/** + * Adust a preexisting token bucket to respect the new configuration + * <b>cfg</b>, by decreasing its current level if needed. */ +void +token_bucket_raw_adjust(token_bucket_raw_t *bucket, + const token_bucket_cfg_t *cfg) +{ + bucket->bucket = MIN(bucket->bucket, cfg->burst); +} + +/** + * Given an amount of <b>elapsed</b> time units, and a bucket configuration + * <b>cfg</b>, refill the level of <b>bucket</b> accordingly. Note that the + * units of time in <b>elapsed</b> must correspond to those used to set the + * rate in <b>cfg</b>, or the result will be illogical. + */ +int +token_bucket_raw_refill_steps(token_bucket_raw_t *bucket, + const token_bucket_cfg_t *cfg, + const uint32_t elapsed) +{ + const int was_empty = (bucket->bucket <= 0); + /* The casts here prevent an underflow. + * + * Note that even if the bucket value is negative, subtracting it from + * "burst" will still produce a correct result. If this result is + * ridiculously high, then the "elapsed > gap / rate" check below + * should catch it. */ + const size_t gap = ((size_t)cfg->burst) - ((size_t)bucket->bucket); + + if (elapsed > gap / cfg->rate) { + bucket->bucket = cfg->burst; + } else { + bucket->bucket += cfg->rate * elapsed; + } + + return was_empty && bucket->bucket > 0; +} + +/** + * Decrement a provided bucket by <b>n</b> units. Note that <b>n</b> + * must be nonnegative. + */ +int +token_bucket_raw_dec(token_bucket_raw_t *bucket, + ssize_t n) +{ + if (BUG(n < 0)) + return 0; + const int becomes_empty = bucket->bucket > 0 && n >= bucket->bucket; + bucket->bucket -= n; + return becomes_empty; +} + +/** Convert a rate in bytes per second to a rate in bytes per step */ +STATIC uint32_t +rate_per_sec_to_rate_per_step(uint32_t rate) +{ + /* + The precise calculation we'd want to do is + + (rate / 1000) * to_approximate_msec(TICKS_PER_STEP). But to minimize + rounding error, we do it this way instead, and divide last. + */ + uint64_t units = (uint64_t) rate * TICKS_PER_STEP; + uint32_t val = (uint32_t) + (monotime_coarse_stamp_units_to_approx_msec(units) / 1000); + return val ? val : 1; +} + +/** + * Initialize a token bucket in *<b>bucket</b>, set up to allow <b>rate</b> + * bytes per second, with a maximum burst of <b>burst</b> bytes. The bucket + * is created such that <b>now_ts</b> is the current timestamp. The bucket + * starts out full. + */ +void +token_bucket_rw_init(token_bucket_rw_t *bucket, + uint32_t rate, + uint32_t burst, + uint32_t now_ts) +{ + memset(bucket, 0, sizeof(token_bucket_rw_t)); + token_bucket_rw_adjust(bucket, rate, burst); + token_bucket_rw_reset(bucket, now_ts); +} + +/** + * Change the configured rate (in bytes per second) and burst (in bytes) + * for the token bucket in *<b>bucket</b>. + */ +void +token_bucket_rw_adjust(token_bucket_rw_t *bucket, + uint32_t rate, + uint32_t burst) +{ + token_bucket_cfg_init(&bucket->cfg, + rate_per_sec_to_rate_per_step(rate), + burst); + token_bucket_raw_adjust(&bucket->read_bucket, &bucket->cfg); + token_bucket_raw_adjust(&bucket->write_bucket, &bucket->cfg); +} + +/** + * Reset <b>bucket</b> to be full, as of timestamp <b>now_ts</b>. + */ +void +token_bucket_rw_reset(token_bucket_rw_t *bucket, + uint32_t now_ts) +{ + token_bucket_raw_reset(&bucket->read_bucket, &bucket->cfg); + token_bucket_raw_reset(&bucket->write_bucket, &bucket->cfg); + bucket->last_refilled_at_timestamp = now_ts; +} + +/** + * Refill <b>bucket</b> as appropriate, given that the current timestamp + * is <b>now_ts</b>. + * + * Return a bitmask containing TB_READ iff read bucket was empty and became + * nonempty, and TB_WRITE iff the write bucket was empty and became nonempty. + */ +int +token_bucket_rw_refill(token_bucket_rw_t *bucket, + uint32_t now_ts) +{ + const uint32_t elapsed_ticks = + (now_ts - bucket->last_refilled_at_timestamp); + if (elapsed_ticks > UINT32_MAX-(300*1000)) { + /* Either about 48 days have passed since the last refill, or the + * monotonic clock has somehow moved backwards. (We're looking at you, + * Windows.). We accept up to a 5 minute jump backwards as + * "unremarkable". + */ + return 0; + } + const uint32_t elapsed_steps = elapsed_ticks / TICKS_PER_STEP; + + if (!elapsed_steps) { + /* Note that if less than one whole step elapsed, we don't advance the + * time in last_refilled_at. That's intentional: we want to make sure + * that we add some bytes to it eventually. */ + return 0; + } + + int flags = 0; + if (token_bucket_raw_refill_steps(&bucket->read_bucket, + &bucket->cfg, elapsed_steps)) + flags |= TB_READ; + if (token_bucket_raw_refill_steps(&bucket->write_bucket, + &bucket->cfg, elapsed_steps)) + flags |= TB_WRITE; + + bucket->last_refilled_at_timestamp = now_ts; + return flags; +} + +/** + * Decrement the read token bucket in <b>bucket</b> by <b>n</b> bytes. + * + * Return true if the bucket was nonempty and became empty; return false + * otherwise. + */ +int +token_bucket_rw_dec_read(token_bucket_rw_t *bucket, + ssize_t n) +{ + return token_bucket_raw_dec(&bucket->read_bucket, n); +} + +/** + * Decrement the write token bucket in <b>bucket</b> by <b>n</b> bytes. + * + * Return true if the bucket was nonempty and became empty; return false + * otherwise. + */ +int +token_bucket_rw_dec_write(token_bucket_rw_t *bucket, + ssize_t n) +{ + return token_bucket_raw_dec(&bucket->write_bucket, n); +} + +/** + * As token_bucket_rw_dec_read and token_bucket_rw_dec_write, in a single + * operation. Return a bitmask of TB_READ and TB_WRITE to indicate + * which buckets became empty. + */ +int +token_bucket_rw_dec(token_bucket_rw_t *bucket, + ssize_t n_read, ssize_t n_written) +{ + int flags = 0; + if (token_bucket_rw_dec_read(bucket, n_read)) + flags |= TB_READ; + if (token_bucket_rw_dec_write(bucket, n_written)) + flags |= TB_WRITE; + return flags; +} + diff --git a/src/common/token_bucket.h b/src/common/token_bucket.h new file mode 100644 index 0000000000..0e7832e838 --- /dev/null +++ b/src/common/token_bucket.h @@ -0,0 +1,118 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file token_bucket_rw.h + * \brief Headers for token_bucket_rw.c + **/ + +#ifndef TOR_TOKEN_BUCKET_H +#define TOR_TOKEN_BUCKET_H + +#include "torint.h" +#include "testsupport.h" + +/** Largest allowable burst value for a token buffer. */ +#define TOKEN_BUCKET_MAX_BURST INT32_MAX + +/** A generic token buffer configuration: determines the number of tokens + * added to the bucket in each time unit (the "rate"), and the maximum number + * of tokens in the bucket (the "burst") */ +typedef struct token_bucket_cfg_t { + uint32_t rate; + int32_t burst; +} token_bucket_cfg_t; + +/** A raw token bucket, decoupled from its configuration and timestamp. */ +typedef struct token_bucket_raw_t { + int32_t bucket; +} token_bucket_raw_t; + +void token_bucket_cfg_init(token_bucket_cfg_t *cfg, + uint32_t rate, + uint32_t burst); + +void token_bucket_raw_adjust(token_bucket_raw_t *bucket, + const token_bucket_cfg_t *cfg); + +void token_bucket_raw_reset(token_bucket_raw_t *bucket, + const token_bucket_cfg_t *cfg); + +int token_bucket_raw_dec(token_bucket_raw_t *bucket, + ssize_t n); + +int token_bucket_raw_refill_steps(token_bucket_raw_t *bucket, + const token_bucket_cfg_t *cfg, + const uint32_t elapsed_steps); + +static inline size_t token_bucket_raw_get(const token_bucket_raw_t *bucket); +/** Return the current number of bytes set in a token bucket. */ +static inline size_t +token_bucket_raw_get(const token_bucket_raw_t *bucket) +{ + return bucket->bucket >= 0 ? bucket->bucket : 0; +} + +/** A convenience type containing all the pieces needed for a coupled + * read-bucket and write-bucket that have the same rate limit, and which use + * "timestamp units" (see compat_time.h) for their time. */ +typedef struct token_bucket_rw_t { + token_bucket_cfg_t cfg; + token_bucket_raw_t read_bucket; + token_bucket_raw_t write_bucket; + uint32_t last_refilled_at_timestamp; +} token_bucket_rw_t; + +void token_bucket_rw_init(token_bucket_rw_t *bucket, + uint32_t rate, + uint32_t burst, + uint32_t now_ts); + +void token_bucket_rw_adjust(token_bucket_rw_t *bucket, + uint32_t rate, uint32_t burst); + +void token_bucket_rw_reset(token_bucket_rw_t *bucket, + uint32_t now_ts); + +#define TB_READ 1 +#define TB_WRITE 2 + +int token_bucket_rw_refill(token_bucket_rw_t *bucket, + uint32_t now_ts); + +int token_bucket_rw_dec_read(token_bucket_rw_t *bucket, + ssize_t n); +int token_bucket_rw_dec_write(token_bucket_rw_t *bucket, + ssize_t n); + +int token_bucket_rw_dec(token_bucket_rw_t *bucket, + ssize_t n_read, ssize_t n_written); + +static inline size_t token_bucket_rw_get_read(const token_bucket_rw_t *bucket); +static inline size_t +token_bucket_rw_get_read(const token_bucket_rw_t *bucket) +{ + return token_bucket_raw_get(&bucket->read_bucket); +} + +static inline size_t token_bucket_rw_get_write( + const token_bucket_rw_t *bucket); +static inline size_t +token_bucket_rw_get_write(const token_bucket_rw_t *bucket) +{ + return token_bucket_raw_get(&bucket->write_bucket); +} + +#ifdef TOKEN_BUCKET_PRIVATE + +/* To avoid making the rates too small, we consider units of "steps", + * where a "step" is defined as this many timestamp ticks. Keep this + * a power of two if you can. */ +#define TICKS_PER_STEP 16 + +STATIC uint32_t rate_per_sec_to_rate_per_step(uint32_t rate); + +#endif + +#endif /* TOR_TOKEN_BUCKET_H */ + diff --git a/src/common/torint.h b/src/common/torint.h index 0b8061d24f..fc7818fe2c 100644 --- a/src/common/torint.h +++ b/src/common/torint.h @@ -40,6 +40,8 @@ #include <inttypes.h> #endif +#include <stdbool.h> + #if (SIZEOF_INT8_T != 0) #define HAVE_INT8_T #endif diff --git a/src/common/torlog.h b/src/common/torlog.h index cadfe3b879..de389883c0 100644 --- a/src/common/torlog.h +++ b/src/common/torlog.h @@ -154,6 +154,8 @@ int add_android_log(const log_severity_list_t *severity, const char *android_identity_tag); #endif // HAVE_ANDROID_LOG_H. int add_callback_log(const log_severity_list_t *severity, log_callback cb); +typedef void (*pending_callback_callback)(void); +void logs_set_pending_callback_callback(pending_callback_callback cb); void logs_set_domain_logging(int enabled); int get_min_log_level(void); void switch_logs_debug(void); @@ -191,6 +193,10 @@ void log_fn_ratelim_(struct ratelim_t *ratelim, int severity, const char *format, ...) CHECK_PRINTF(5,6); +int log_message_is_interesting(int severity, log_domain_mask_t domain); +void tor_log_string(int severity, log_domain_mask_t domain, + const char *function, const char *string); + #if defined(__GNUC__) && __GNUC__ <= 3 /* These are the GCC varidaic macros, so that older versions of GCC don't @@ -248,6 +254,16 @@ void log_fn_ratelim_(struct ratelim_t *ratelim, int severity, args, ##__VA_ARGS__) #endif /* defined(__GNUC__) && __GNUC__ <= 3 */ +/** This defines log levels that are linked in the Rust log module, rather + * than re-defining these in both Rust and C. + * + * C_RUST_COUPLED src/rust/tor_log LogSeverity, LogDomain + */ +extern const int LOG_WARN_; +extern const int LOG_NOTICE_; +extern const log_domain_mask_t LD_NET_; +extern const log_domain_mask_t LD_GENERAL_; + #ifdef LOG_PRIVATE MOCK_DECL(STATIC void, logv, (int severity, log_domain_mask_t domain, const char *funcname, const char *suffix, const char *format, diff --git a/src/common/tortls.c b/src/common/tortls.c index 4ceb38ac86..653bd66de5 100644 --- a/src/common/tortls.c +++ b/src/common/tortls.c @@ -25,6 +25,9 @@ #include <ws2tcpip.h> #endif +#include "crypto.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "compat.h" /* Some versions of OpenSSL declare SSL_get_selected_srtp_profile twice in @@ -32,7 +35,6 @@ DISABLE_GCC_WARNING(redundant-decls) #include <openssl/opensslv.h> -#include "crypto.h" #ifdef OPENSSL_NO_EC #error "We require OpenSSL with ECC support" @@ -56,10 +58,25 @@ ENABLE_GCC_WARNING(redundant-decls) #include "container.h" #include <string.h> +#ifdef OPENSSL_1_1_API +#define X509_get_notBefore_const(cert) \ + X509_get0_notBefore(cert) +#define X509_get_notAfter_const(cert) \ + X509_get0_notAfter(cert) +#ifndef X509_get_notBefore +#define X509_get_notBefore(cert) \ + X509_getm_notBefore(cert) +#endif +#ifndef X509_get_notAfter +#define X509_get_notAfter(cert) \ + X509_getm_notAfter(cert) +#endif +#else /* ! OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0) */ #define X509_get_notBefore_const(cert) \ ((const ASN1_TIME*) X509_get_notBefore((X509 *)cert)) #define X509_get_notAfter_const(cert) \ ((const ASN1_TIME*) X509_get_notAfter((X509 *)cert)) +#endif /* Copied from or.h */ #define LEGAL_NICKNAME_CHARACTERS \ @@ -355,8 +372,12 @@ tor_tls_init(void) check_no_tls_errors(); if (!tls_library_is_initialized) { +#ifdef OPENSSL_1_1_API + OPENSSL_init_ssl(OPENSSL_INIT_LOAD_SSL_STRINGS, NULL); +#else SSL_library_init(); SSL_load_error_strings(); +#endif #if (SIZEOF_VOID_P >= 8 && \ OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,0,1)) diff --git a/src/common/tortls.h b/src/common/tortls.h index 1dbf0b332f..7c867bfff2 100644 --- a/src/common/tortls.h +++ b/src/common/tortls.h @@ -11,7 +11,7 @@ * \brief Headers for tortls.c **/ -#include "crypto.h" +#include "crypto_rsa.h" #include "compat_openssl.h" #include "compat.h" #include "testsupport.h" diff --git a/src/common/util.c b/src/common/util.c index a68fd30d09..dece5877f1 100644 --- a/src/common/util.c +++ b/src/common/util.c @@ -16,7 +16,7 @@ #define UTIL_PRIVATE #include "util.h" #include "torlog.h" -#include "crypto.h" +#include "crypto_digest.h" #include "torint.h" #include "container.h" #include "address.h" @@ -572,6 +572,19 @@ add_laplace_noise(int64_t signal_, double random_, double delta_f, return signal_ + noise; } +/* Helper: safely add two uint32_t's, capping at UINT32_MAX rather + * than overflow */ +uint32_t +tor_add_u32_nowrap(uint32_t a, uint32_t b) +{ + /* a+b > UINT32_MAX check, without overflow */ + if (PREDICT_UNLIKELY(a > UINT32_MAX - b)) { + return UINT32_MAX; + } else { + return a+b; + } +} + /* Helper: return greatest common divisor of a,b */ static uint64_t gcd64(uint64_t a, uint64_t b) @@ -1821,6 +1834,15 @@ format_iso_time(char *buf, time_t t) strftime(buf, ISO_TIME_LEN+1, "%Y-%m-%d %H:%M:%S", tor_gmtime_r(&t, &tm)); } +/** As format_local_iso_time, but use the yyyy-mm-ddThh:mm:ss format to avoid + * embedding an internal space. */ +void +format_local_iso_time_nospace(char *buf, time_t t) +{ + format_local_iso_time(buf, t); + buf[10] = 'T'; +} + /** As format_iso_time, but use the yyyy-mm-ddThh:mm:ss format to avoid * embedding an internal space. */ void @@ -4779,8 +4801,8 @@ process_environment_t * process_environment_make(struct smartlist_t *env_vars) { process_environment_t *env = tor_malloc_zero(sizeof(process_environment_t)); - size_t n_env_vars = smartlist_len(env_vars); - size_t i; + int n_env_vars = smartlist_len(env_vars); + int i; size_t total_env_length; smartlist_t *env_vars_sorted; @@ -5111,30 +5133,6 @@ stream_status_to_string(enum stream_status stream_status) } } -/* DOCDOC */ -static void -log_portfw_spawn_error_message(const char *buf, - const char *executable, int *child_status) -{ - /* Parse error message */ - int retval, child_state, saved_errno; - retval = tor_sscanf(buf, SPAWN_ERROR_MESSAGE "%x/%x", - &child_state, &saved_errno); - if (retval == 2) { - log_warn(LD_GENERAL, - "Failed to start child process \"%s\" in state %d: %s", - executable, child_state, strerror(saved_errno)); - if (child_status) - *child_status = 1; - } else { - /* Failed to parse message from child process, log it as a - warning */ - log_warn(LD_GENERAL, - "Unexpected message from port forwarding helper \"%s\": %s", - executable, buf); - } -} - #ifdef _WIN32 /** Return a smartlist containing lines outputted from @@ -5180,51 +5178,6 @@ tor_get_lines_from_handle, (HANDLE *handle, return lines; } -/** Read from stream, and send lines to log at the specified log level. - * Returns -1 if there is a error reading, and 0 otherwise. - * If the generated stream is flushed more often than on new lines, or - * a read exceeds 256 bytes, lines will be truncated. This should be fixed, - * along with the corresponding problem on *nix (see bug #2045). - */ -static int -log_from_handle(HANDLE *pipe, int severity) -{ - char buf[256]; - int pos; - smartlist_t *lines; - - pos = tor_read_all_handle(pipe, buf, sizeof(buf) - 1, NULL); - if (pos < 0) { - /* Error */ - log_warn(LD_GENERAL, "Failed to read data from subprocess"); - return -1; - } - - if (0 == pos) { - /* There's nothing to read (process is busy or has exited) */ - log_debug(LD_GENERAL, "Subprocess had nothing to say"); - return 0; - } - - /* End with a null even if there isn't a \r\n at the end */ - /* TODO: What if this is a partial line? */ - buf[pos] = '\0'; - log_debug(LD_GENERAL, "Subprocess had %d bytes to say", pos); - - /* Split up the buffer */ - lines = smartlist_new(); - tor_split_lines(lines, buf, pos); - - /* Log each line */ - SMARTLIST_FOREACH(lines, char *, line, - { - log_fn(severity, LD_GENERAL, "Port forwarding helper says: %s", line); - }); - smartlist_free(lines); - - return 0; -} - #else /* !(defined(_WIN32)) */ /** Return a smartlist containing lines outputted from @@ -5254,42 +5207,6 @@ tor_get_lines_from_handle, (int fd, enum stream_status *stream_status_out)) return lines; } -/** Read from fd, and send lines to log at the specified log level. - * Returns 1 if stream is closed normally, -1 if there is a error reading, and - * 0 otherwise. Handles lines from tor-fw-helper and - * tor_spawn_background() specially. - */ -static int -log_from_pipe(int fd, int severity, const char *executable, - int *child_status) -{ - char buf[256]; - enum stream_status r; - - for (;;) { - r = get_string_from_pipe(fd, buf, sizeof(buf) - 1); - - if (r == IO_STREAM_CLOSED) { - return 1; - } else if (r == IO_STREAM_EAGAIN) { - return 0; - } else if (r == IO_STREAM_TERM) { - return -1; - } - - tor_assert(r == IO_STREAM_OKAY); - - /* Check if buf starts with SPAWN_ERROR_MESSAGE */ - if (strcmpstart(buf, SPAWN_ERROR_MESSAGE) == 0) { - log_portfw_spawn_error_message(buf, executable, child_status); - } else { - log_fn(severity, LD_GENERAL, "Port forwarding helper says: %s", buf); - } - } - - /* We should never get here */ - return -1; -} #endif /* defined(_WIN32) */ /** Reads from <b>fd</b> and stores input in <b>buf_out</b> making @@ -5332,294 +5249,6 @@ get_string_from_pipe(int fd, char *buf_out, size_t count) return IO_STREAM_OKAY; } -/** Parse a <b>line</b> from tor-fw-helper and issue an appropriate - * log message to our user. */ -static void -handle_fw_helper_line(const char *executable, const char *line) -{ - smartlist_t *tokens = smartlist_new(); - char *message = NULL; - char *message_for_log = NULL; - const char *external_port = NULL; - const char *internal_port = NULL; - const char *result = NULL; - int port = 0; - int success = 0; - - if (strcmpstart(line, SPAWN_ERROR_MESSAGE) == 0) { - /* We need to check for SPAWN_ERROR_MESSAGE again here, since it's - * possible that it got sent after we tried to read it in log_from_pipe. - * - * XXX Ideally, we should be using one of stdout/stderr for the real - * output, and one for the output of the startup code. We used to do that - * before cd05f35d2c. - */ - int child_status; - log_portfw_spawn_error_message(line, executable, &child_status); - goto done; - } - - smartlist_split_string(tokens, line, NULL, - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1); - - if (smartlist_len(tokens) < 5) - goto err; - - if (strcmp(smartlist_get(tokens, 0), "tor-fw-helper") || - strcmp(smartlist_get(tokens, 1), "tcp-forward")) - goto err; - - external_port = smartlist_get(tokens, 2); - internal_port = smartlist_get(tokens, 3); - result = smartlist_get(tokens, 4); - - if (smartlist_len(tokens) > 5) { - /* If there are more than 5 tokens, they are part of [<message>]. - Let's use a second smartlist to form the whole message; - strncat loops suck. */ - int i; - int message_words_n = smartlist_len(tokens) - 5; - smartlist_t *message_sl = smartlist_new(); - for (i = 0; i < message_words_n; i++) - smartlist_add(message_sl, smartlist_get(tokens, 5+i)); - - tor_assert(smartlist_len(message_sl) > 0); - message = smartlist_join_strings(message_sl, " ", 0, NULL); - - /* wrap the message in log-friendly wrapping */ - tor_asprintf(&message_for_log, " ('%s')", message); - - smartlist_free(message_sl); - } - - port = atoi(external_port); - if (port < 1 || port > 65535) - goto err; - - port = atoi(internal_port); - if (port < 1 || port > 65535) - goto err; - - if (!strcmp(result, "SUCCESS")) - success = 1; - else if (!strcmp(result, "FAIL")) - success = 0; - else - goto err; - - if (!success) { - log_warn(LD_GENERAL, "Tor was unable to forward TCP port '%s' to '%s'%s. " - "Please make sure that your router supports port " - "forwarding protocols (like NAT-PMP). Note that if '%s' is " - "your ORPort, your relay will be unable to receive inbound " - "traffic.", external_port, internal_port, - message_for_log ? message_for_log : "", - internal_port); - } else { - log_info(LD_GENERAL, - "Tor successfully forwarded TCP port '%s' to '%s'%s.", - external_port, internal_port, - message_for_log ? message_for_log : ""); - } - - goto done; - - err: - log_warn(LD_GENERAL, "tor-fw-helper sent us a string we could not " - "parse (%s).", line); - - done: - SMARTLIST_FOREACH(tokens, char *, cp, tor_free(cp)); - smartlist_free(tokens); - tor_free(message); - tor_free(message_for_log); -} - -/** Read what tor-fw-helper has to say in its stdout and handle it - * appropriately */ -static int -handle_fw_helper_output(const char *executable, - process_handle_t *process_handle) -{ - smartlist_t *fw_helper_output = NULL; - enum stream_status stream_status = 0; - - fw_helper_output = - tor_get_lines_from_handle(tor_process_get_stdout_pipe(process_handle), - &stream_status); - if (!fw_helper_output) { /* didn't get any output from tor-fw-helper */ - /* if EAGAIN we should retry in the future */ - return (stream_status == IO_STREAM_EAGAIN) ? 0 : -1; - } - - /* Handle the lines we got: */ - SMARTLIST_FOREACH_BEGIN(fw_helper_output, char *, line) { - handle_fw_helper_line(executable, line); - tor_free(line); - } SMARTLIST_FOREACH_END(line); - - smartlist_free(fw_helper_output); - - return 0; -} - -/** Spawn tor-fw-helper and ask it to forward the ports in - * <b>ports_to_forward</b>. <b>ports_to_forward</b> contains strings - * of the form "<external port>:<internal port>", which is the format - * that tor-fw-helper expects. */ -void -tor_check_port_forwarding(const char *filename, - smartlist_t *ports_to_forward, - time_t now) -{ -/* When fw-helper succeeds, how long do we wait until running it again */ -#define TIME_TO_EXEC_FWHELPER_SUCCESS 300 -/* When fw-helper failed to start, how long do we wait until running it again - */ -#define TIME_TO_EXEC_FWHELPER_FAIL 60 - - /* Static variables are initialized to zero, so child_handle.status=0 - * which corresponds to it not running on startup */ - static process_handle_t *child_handle=NULL; - - static time_t time_to_run_helper = 0; - int stderr_status, retval; - int stdout_status = 0; - - tor_assert(filename); - - /* Start the child, if it is not already running */ - if ((!child_handle || child_handle->status != PROCESS_STATUS_RUNNING) && - time_to_run_helper < now) { - /*tor-fw-helper cli looks like this: tor_fw_helper -p :5555 -p 4555:1111 */ - const char **argv; /* cli arguments */ - int args_n, status; - int argv_index = 0; /* index inside 'argv' */ - - tor_assert(smartlist_len(ports_to_forward) > 0); - - /* check for overflow during 'argv' allocation: - (len(ports_to_forward)*2 + 2)*sizeof(char*) > SIZE_MAX == - len(ports_to_forward) > (((SIZE_MAX/sizeof(char*)) - 2)/2) */ - if ((size_t) smartlist_len(ports_to_forward) > - (((SIZE_MAX/sizeof(char*)) - 2)/2)) { - log_warn(LD_GENERAL, - "Overflow during argv allocation. This shouldn't happen."); - return; - } - /* check for overflow during 'argv_index' increase: - ((len(ports_to_forward)*2 + 2) > INT_MAX) == - len(ports_to_forward) > (INT_MAX - 2)/2 */ - if (smartlist_len(ports_to_forward) > (INT_MAX - 2)/2) { - log_warn(LD_GENERAL, - "Overflow during argv_index increase. This shouldn't happen."); - return; - } - - /* Calculate number of cli arguments: one for the filename, two - for each smartlist element (one for "-p" and one for the - ports), and one for the final NULL. */ - args_n = 1 + 2*smartlist_len(ports_to_forward) + 1; - argv = tor_calloc(args_n, sizeof(char *)); - - argv[argv_index++] = filename; - SMARTLIST_FOREACH_BEGIN(ports_to_forward, const char *, port) { - argv[argv_index++] = "-p"; - argv[argv_index++] = port; - } SMARTLIST_FOREACH_END(port); - argv[argv_index] = NULL; - - /* Assume tor-fw-helper will succeed, start it later*/ - time_to_run_helper = now + TIME_TO_EXEC_FWHELPER_SUCCESS; - - if (child_handle) { - tor_process_handle_destroy(child_handle, 1); - child_handle = NULL; - } - -#ifdef _WIN32 - /* Passing NULL as lpApplicationName makes Windows search for the .exe */ - status = tor_spawn_background(NULL, argv, NULL, &child_handle); -#else - status = tor_spawn_background(filename, argv, NULL, &child_handle); -#endif /* defined(_WIN32) */ - - tor_free_((void*)argv); - argv=NULL; - - if (PROCESS_STATUS_ERROR == status) { - log_warn(LD_GENERAL, "Failed to start port forwarding helper %s", - filename); - time_to_run_helper = now + TIME_TO_EXEC_FWHELPER_FAIL; - return; - } - - log_info(LD_GENERAL, - "Started port forwarding helper (%s) with pid '%d'", - filename, tor_process_get_pid(child_handle)); - } - - /* If child is running, read from its stdout and stderr) */ - if (child_handle && PROCESS_STATUS_RUNNING == child_handle->status) { - /* Read from stdout/stderr and log result */ - retval = 0; -#ifdef _WIN32 - stderr_status = log_from_handle(child_handle->stderr_pipe, LOG_INFO); -#else - stderr_status = log_from_pipe(child_handle->stderr_pipe, - LOG_INFO, filename, &retval); -#endif /* defined(_WIN32) */ - if (handle_fw_helper_output(filename, child_handle) < 0) { - log_warn(LD_GENERAL, "Failed to handle fw helper output."); - stdout_status = -1; - retval = -1; - } - - if (retval) { - /* There was a problem in the child process */ - time_to_run_helper = now + TIME_TO_EXEC_FWHELPER_FAIL; - } - - /* Combine the two statuses in order of severity */ - if (-1 == stdout_status || -1 == stderr_status) - /* There was a failure */ - retval = -1; -#ifdef _WIN32 - else if (!child_handle || tor_get_exit_code(child_handle, 0, NULL) != - PROCESS_EXIT_RUNNING) { - /* process has exited or there was an error */ - /* TODO: Do something with the process return value */ - /* TODO: What if the process output something since - * between log_from_handle and tor_get_exit_code? */ - retval = 1; - } -#else /* !(defined(_WIN32)) */ - else if (1 == stdout_status || 1 == stderr_status) - /* stdout or stderr was closed, the process probably - * exited. It will be reaped by waitpid() in main.c */ - /* TODO: Do something with the process return value */ - retval = 1; -#endif /* defined(_WIN32) */ - else - /* Both are fine */ - retval = 0; - - /* If either pipe indicates a failure, act on it */ - if (0 != retval) { - if (1 == retval) { - log_info(LD_GENERAL, "Port forwarding helper terminated"); - child_handle->status = PROCESS_STATUS_NOTRUNNING; - } else { - log_warn(LD_GENERAL, "Failed to read from port forwarding helper"); - child_handle->status = PROCESS_STATUS_ERROR; - } - - /* TODO: The child might not actually be finished (maybe it failed or - closed stdout/stderr), so maybe we shouldn't start another? */ - } - } -} - /** Initialize the insecure RNG <b>rng</b> from a seed value <b>seed</b>. */ void tor_init_weak_random(tor_weak_rng_t *rng, unsigned seed) diff --git a/src/common/util.h b/src/common/util.h index 9380789128..7172b7da08 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -73,9 +73,9 @@ extern int dmalloc_free(const char *file, const int line, void *pnt, } \ STMT_END #else /* !(defined(USE_DMALLOC)) */ -/** Release memory allocated by tor_malloc, tor_realloc, tor_strdup, etc. - * Unlike the free() function, tor_free() will still work on NULL pointers, - * and it sets the pointer value to NULL after freeing it. +/** Release memory allocated by tor_malloc, tor_realloc, tor_strdup, + * etc. Unlike the free() function, the tor_free() macro sets the + * pointer value to NULL after freeing it. * * This is a macro. If you need a function pointer to release memory from * tor_malloc(), use tor_free_(). @@ -88,17 +88,13 @@ extern int dmalloc_free(const char *file, const int line, void *pnt, #ifdef __GNUC__ #define tor_free(p) STMT_BEGIN \ typeof(&(p)) tor_free__tmpvar = &(p); \ - if (PREDICT_LIKELY((*tor_free__tmpvar)!=NULL)) { \ - raw_free(*tor_free__tmpvar); \ - *tor_free__tmpvar=NULL; \ - } \ + raw_free(*tor_free__tmpvar); \ + *tor_free__tmpvar=NULL; \ STMT_END #else #define tor_free(p) STMT_BEGIN \ - if (PREDICT_LIKELY((p)!=NULL)) { \ - raw_free(p); \ - (p)=NULL; \ - } \ + raw_free(p); \ + (p)=NULL; \ STMT_END #endif #endif /* defined(USE_DMALLOC) */ @@ -180,6 +176,8 @@ int n_bits_set_u8(uint8_t v); int64_t clamp_double_to_int64(double number); void simplify_fraction64(uint64_t *numer, uint64_t *denom); +uint32_t tor_add_u32_nowrap(uint32_t a, uint32_t b); + /* Compute the CEIL of <b>a</b> divided by <b>b</b>, for nonnegative <b>a</b> * and positive <b>b</b>. Works on integer types only. Not defined if a+(b-1) * can overflow. */ @@ -273,6 +271,7 @@ int parse_rfc1123_time(const char *buf, time_t *t); #define ISO_TIME_USEC_LEN (ISO_TIME_LEN+7) void format_local_iso_time(char *buf, time_t t); void format_iso_time(char *buf, time_t t); +void format_local_iso_time_nospace(char *buf, time_t t); void format_iso_time_nospace(char *buf, time_t t); void format_iso_time_nospace_usec(char *buf, const struct timeval *tv); int parse_iso_time_(const char *cp, time_t *t, int strict, int nospace); @@ -418,11 +417,6 @@ void start_daemon(void); void finish_daemon(const char *desired_cwd); int write_pidfile(const char *filename); -/* Port forwarding */ -void tor_check_port_forwarding(const char *filename, - struct smartlist_t *ports_to_forward, - time_t now); - void tor_disable_spawning_background_processes(void); typedef struct process_handle_t process_handle_t; @@ -461,9 +455,7 @@ void set_environment_variable_in_smartlist(struct smartlist_t *env_vars, void (*free_old)(void*), int free_p); -/* Values of process_handle_t.status. PROCESS_STATUS_NOTRUNNING must be - * 0 because tor_check_port_forwarding depends on this being the initial - * statue of the static instance of process_handle_t */ +/* Values of process_handle_t.status. */ #define PROCESS_STATUS_NOTRUNNING 0 #define PROCESS_STATUS_RUNNING 1 #define PROCESS_STATUS_ERROR -1 diff --git a/src/common/workqueue.c b/src/common/workqueue.c index ec96959b7d..563a98af96 100644 --- a/src/common/workqueue.c +++ b/src/common/workqueue.c @@ -1,3 +1,4 @@ + /* copyright (c) 2013-2015, The Tor Project, Inc. */ /* See LICENSE for licensing information */ @@ -24,13 +25,16 @@ #include "orconfig.h" #include "compat.h" +#include "compat_libevent.h" #include "compat_threads.h" -#include "crypto.h" +#include "crypto_rand.h" #include "util.h" #include "workqueue.h" #include "tor_queue.h" #include "torlog.h" +#include <event2/event.h> + #define WORKQUEUE_PRIORITY_FIRST WQ_PRI_HIGH #define WORKQUEUE_PRIORITY_LAST WQ_PRI_LOW #define WORKQUEUE_N_PRIORITIES (((int) WORKQUEUE_PRIORITY_LAST)+1) @@ -63,6 +67,9 @@ struct threadpool_s { void (*free_update_arg_fn)(void *); /** Array of n_threads update arguments. */ void **update_args; + /** Event to notice when another thread has sent a reply. */ + struct event *reply_event; + void (*reply_cb)(threadpool_t *); /** Number of elements in threads. */ int n_threads; @@ -597,15 +604,41 @@ replyqueue_new(uint32_t alertsocks_flags) return rq; } -/** - * Return the "read socket" for a given reply queue. The main thread should - * listen for read events on this socket, and call replyqueue_process() every - * time it triggers. +/** Internal: Run from the libevent mainloop when there is work to handle in + * the reply queue handler. */ +static void +reply_event_cb(evutil_socket_t sock, short events, void *arg) +{ + threadpool_t *tp = arg; + (void) sock; + (void) events; + replyqueue_process(tp->reply_queue); + if (tp->reply_cb) + tp->reply_cb(tp); +} + +/** Register the threadpool <b>tp</b>'s reply queue with the libevent + * mainloop of <b>base</b>. If <b>tp</b> is provided, it is run after + * each time there is work to process from the reply queue. Return 0 on + * success, -1 on failure. */ -tor_socket_t -replyqueue_get_socket(replyqueue_t *rq) +int +threadpool_register_reply_event(threadpool_t *tp, + void (*cb)(threadpool_t *tp)) { - return rq->alert.read_fd; + struct event_base *base = tor_libevent_get_base(); + + if (tp->reply_event) { + tor_event_free(tp->reply_event); + } + tp->reply_event = tor_event_new(base, + tp->reply_queue->alert.read_fd, + EV_READ|EV_PERSIST, + reply_event_cb, + tp); + tor_assert(tp->reply_event); + tp->reply_cb = cb; + return event_add(tp->reply_event, NULL); } /** diff --git a/src/common/workqueue.h b/src/common/workqueue.h index eb885e680d..e1fe612e2b 100644 --- a/src/common/workqueue.h +++ b/src/common/workqueue.h @@ -56,8 +56,11 @@ threadpool_t *threadpool_new(int n_threads, replyqueue_t *threadpool_get_replyqueue(threadpool_t *tp); replyqueue_t *replyqueue_new(uint32_t alertsocks_flags); -tor_socket_t replyqueue_get_socket(replyqueue_t *rq); void replyqueue_process(replyqueue_t *queue); +struct event_base; +int threadpool_register_reply_event(threadpool_t *tp, + void (*cb)(threadpool_t *tp)); + #endif /* !defined(TOR_WORKQUEUE_H) */ diff --git a/src/config/torrc.minimal.in-staging b/src/config/torrc.minimal.in-staging index 90f91e5cb9..86429f1176 100644 --- a/src/config/torrc.minimal.in-staging +++ b/src/config/torrc.minimal.in-staging @@ -100,7 +100,7 @@ ## A handle for your relay, so people don't have to refer to it by key. ## Nicknames must be between 1 and 19 characters inclusive, and must ## contain only the alphanumeric characters (a-z, A-Z, 0-9). No unicode, -## no emoji. +## no emoji. If not set, "Unnamed" will be used. #Nickname ididnteditheconfig ## Define these to limit how much relayed traffic you will allow. Your diff --git a/src/config/torrc.sample.in b/src/config/torrc.sample.in index 4e183478eb..72cca0be31 100644 --- a/src/config/torrc.sample.in +++ b/src/config/torrc.sample.in @@ -106,6 +106,7 @@ ## A handle for your relay, so people don't have to refer to it by key. ## Nicknames must be between 1 and 19 characters inclusive, and must ## contain only the characters [a-zA-Z0-9]. +## If not set, "Unnamed" will be used. #Nickname ididnteditheconfig ## Define these to limit how much relayed traffic you will allow. Your diff --git a/src/ext/ed25519/donna/ed25519-hash-custom.h b/src/ext/ed25519/donna/ed25519-hash-custom.h index 609451abd5..cdeab3e45b 100644 --- a/src/ext/ed25519/donna/ed25519-hash-custom.h +++ b/src/ext/ed25519/donna/ed25519-hash-custom.h @@ -9,7 +9,7 @@ void ed25519_hash(uint8_t *hash, const uint8_t *in, size_t inlen); */ -#include "crypto.h" +#include "crypto_digest.h" typedef struct ed25519_hash_context { crypto_digest_t *ctx; diff --git a/src/ext/ed25519/donna/ed25519-randombytes-custom.h b/src/ext/ed25519/donna/ed25519-randombytes-custom.h index 3fb0959fc4..27eade4f95 100644 --- a/src/ext/ed25519/donna/ed25519-randombytes-custom.h +++ b/src/ext/ed25519/donna/ed25519-randombytes-custom.h @@ -8,7 +8,7 @@ */ /* Tor: Instead of calling OpenSSL's CSPRNG directly, call the wrapper. */ -#include "crypto.h" +#include "crypto_rand.h" static void ED25519_FN(ed25519_randombytes_unsafe) (void *p, size_t len) diff --git a/src/ext/ed25519/donna/ed25519_tor.c b/src/ext/ed25519/donna/ed25519_tor.c index 84fc3850a2..43de9faaea 100644 --- a/src/ext/ed25519/donna/ed25519_tor.c +++ b/src/ext/ed25519/donna/ed25519_tor.c @@ -40,6 +40,8 @@ #include "ed25519-randombytes.h" #include "ed25519-hash.h" +#include "crypto_util.h" + typedef unsigned char ed25519_signature[64]; typedef unsigned char ed25519_public_key[32]; typedef unsigned char ed25519_secret_key[32]; diff --git a/src/ext/ed25519/ref10/blinding.c b/src/ext/ed25519/ref10/blinding.c index a3b32fa80c..88e84cac20 100644 --- a/src/ext/ed25519/ref10/blinding.c +++ b/src/ext/ed25519/ref10/blinding.c @@ -7,7 +7,7 @@ #include "ed25519_ref10.h" #include <string.h> -#include "crypto.h" +#include "crypto_util.h" static void ed25519_ref10_gettweak(unsigned char *out, const unsigned char *param) diff --git a/src/ext/ed25519/ref10/crypto_hash_sha512.h b/src/ext/ed25519/ref10/crypto_hash_sha512.h index 5dad935c79..7faddb1597 100644 --- a/src/ext/ed25519/ref10/crypto_hash_sha512.h +++ b/src/ext/ed25519/ref10/crypto_hash_sha512.h @@ -1,5 +1,5 @@ /* Added for Tor. */ -#include "crypto.h" +#include "crypto_digest.h" /* Set 'out' to the 512-bit SHA512 hash of the 'len'-byte string in 'inp' */ #define crypto_hash_sha512(out, inp, len) \ diff --git a/src/ext/ed25519/ref10/keypair.c b/src/ext/ed25519/ref10/keypair.c index 68a88f9adc..c437f0a4f2 100644 --- a/src/ext/ed25519/ref10/keypair.c +++ b/src/ext/ed25519/ref10/keypair.c @@ -6,6 +6,9 @@ #include "crypto_hash_sha512.h" #include "ge.h" +#include "crypto_rand.h" +#include "crypto_util.h" + int crypto_sign_seckey(unsigned char *sk) { diff --git a/src/ext/ed25519/ref10/randombytes.h b/src/ext/ed25519/ref10/randombytes.h index 8bf31631f0..a21dde8540 100644 --- a/src/ext/ed25519/ref10/randombytes.h +++ b/src/ext/ed25519/ref10/randombytes.h @@ -1,4 +1,4 @@ /* Added for Tor. */ -#include "crypto.h" +#include "crypto_rand.h" #define randombytes(b, n) \ (crypto_strongest_rand((b), (n)), 0) diff --git a/src/ext/keccak-tiny/keccak-tiny-unrolled.c b/src/ext/keccak-tiny/keccak-tiny-unrolled.c index d8d7fe335a..07e8c95bcf 100644 --- a/src/ext/keccak-tiny/keccak-tiny-unrolled.c +++ b/src/ext/keccak-tiny/keccak-tiny-unrolled.c @@ -9,7 +9,7 @@ #include "keccak-tiny.h" #include <string.h> -#include "crypto.h" +#include "crypto_util.h" #include "byteorder.h" /******** Endianness conversion helpers ********/ diff --git a/src/ext/rust b/src/ext/rust -Subproject fbc0c25785696a25b9cbc09ed645cc8d404ee0f +Subproject aa37fb84fb829902e83ca11a7244bbc6b86b809 diff --git a/src/ext/timeouts/timeout.c b/src/ext/timeouts/timeout.c index 713ec219ce..d4b514d2c5 100644 --- a/src/ext/timeouts/timeout.c +++ b/src/ext/timeouts/timeout.c @@ -150,7 +150,7 @@ #else #define ctz(n) ctz32(n) #define clz(n) clz32(n) -#define fls(n) ((int)(32 - clz32(n))) +#define fls(n) ((int)(32 - clz32((uint32_t)n))) #endif #if WHEEL_BIT == 6 @@ -432,7 +432,7 @@ TIMEOUT_PUBLIC void timeouts_update(struct timeouts *T, abstime_t curtime) { * or can be replaced with a simpler operation. */ oslot = WHEEL_MASK & (T->curtime >> (wheel * WHEEL_BIT)); - pending = rotl(((UINT64_C(1) << _elapsed) - 1), oslot); + pending = rotl(((WHEEL_C(1) << _elapsed) - 1), oslot); nslot = WHEEL_MASK & (curtime >> (wheel * WHEEL_BIT)); pending |= rotr(rotl(((WHEEL_C(1) << _elapsed) - 1), nslot), (int)_elapsed); diff --git a/src/or/addressmap.c b/src/or/addressmap.c index 96ce275578..7f861e4d24 100644 --- a/src/or/addressmap.c +++ b/src/or/addressmap.c @@ -21,9 +21,10 @@ #include "config.h" #include "connection_edge.h" #include "control.h" +#include "crypto_rand.h" #include "dns.h" -#include "routerset.h" #include "nodelist.h" +#include "routerset.h" /** A client-side struct to remember requests to rewrite addresses * to new addresses. These structs are stored in the hash table @@ -959,9 +960,11 @@ addressmap_get_virtual_address(int type) char tmp[TOR_ADDR_BUF_LEN]; tor_addr_to_str(tmp, &addr, sizeof(tmp), 0); if (strmap_get(addressmap, tmp)) { + // LCOV_EXCL_START log_warn(LD_BUG, "%s wasn't in the addressmap, but %s was.", buf, tmp); continue; + // LCOV_EXCL_STOP } return tor_strdup(buf); @@ -970,8 +973,10 @@ addressmap_get_virtual_address(int type) log_warn(LD_CONFIG, "Ran out of virtual addresses!"); return NULL; } else { + // LCOV_EXCL_START log_warn(LD_BUG, "Called with unsupported address type (%d)", type); return NULL; + // LCOV_EXCL_STOP } } diff --git a/src/or/bridges.c b/src/or/bridges.c index 29d00f37ba..699e030e6c 100644 --- a/src/or/bridges.c +++ b/src/or/bridges.c @@ -11,6 +11,8 @@ * Bridges are fixed entry nodes, used for censorship circumvention. **/ +#define TOR_BRIDGES_PRIVATE + #include "or.h" #include "bridges.h" #include "circuitbuild.h" @@ -93,7 +95,7 @@ sweep_bridge_list(void) } /** Initialize the bridge list to empty, creating it if needed. */ -static void +STATIC void clear_bridge_list(void) { if (!bridge_list) @@ -156,7 +158,7 @@ bridge_get_addr_port(const bridge_info_t *bridge) * bridge with no known digest whose address matches any of the * tor_addr_port_t's in <b>orports</b>, return that bridge. Else return * NULL. */ -static bridge_info_t * +STATIC bridge_info_t * get_configured_bridge_by_orports_digest(const char *digest, const smartlist_t *orports) { @@ -350,7 +352,7 @@ bridge_has_digest(const bridge_info_t *bridge, const char *digest) * existing bridge with the same address and port, and warn the user as * appropriate. */ -static void +STATIC void bridge_resolve_conflicts(const tor_addr_t *addr, uint16_t port, const char *digest, const char *transport_name) { @@ -471,7 +473,7 @@ bridge_add_from_config(bridge_line_t *bridge_line) } /** If <b>digest</b> is one of our known bridges, return it. */ -bridge_info_t * +STATIC bridge_info_t * find_bridge_by_digest(const char *digest) { if (! bridge_list) diff --git a/src/or/bridges.h b/src/or/bridges.h index 54a6250259..3108eb555d 100644 --- a/src/or/bridges.h +++ b/src/or/bridges.h @@ -20,7 +20,6 @@ typedef struct bridge_info_t bridge_info_t; void mark_bridge_list(void); void sweep_bridge_list(void); const smartlist_t *bridge_list_get(void); -bridge_info_t *find_bridge_by_digest(const char *digest); const uint8_t *bridge_get_rsa_id_digest(const bridge_info_t *bridge); const tor_addr_port_t * bridge_get_addr_port(const bridge_info_t *bridge); bridge_info_t *get_configured_bridge_by_addr_port_digest( @@ -65,5 +64,17 @@ MOCK_DECL(download_status_t *, get_bridge_dl_status_by_id, void bridges_free_all(void); +#ifdef TOR_BRIDGES_PRIVATE +STATIC void clear_bridge_list(void); +STATIC bridge_info_t *find_bridge_by_digest(const char *digest); +STATIC bridge_info_t *get_configured_bridge_by_orports_digest( + const char *digest, + const smartlist_t *orports); +STATIC void bridge_resolve_conflicts(const tor_addr_t *addr, + uint16_t port, + const char *digest, + const char *transport_name); +#endif /* defined(TOR_BRIDGES_PRIVATE) */ + #endif /* !defined(TOR_BRIDGES_H) */ diff --git a/src/or/channel.c b/src/or/channel.c index a4740dd752..c30e508018 100644 --- a/src/or/channel.c +++ b/src/or/channel.c @@ -69,6 +69,7 @@ #include "circuitmux.h" #include "entrynodes.h" #include "geoip.h" +#include "main.h" #include "nodelist.h" #include "relay.h" #include "rephist.h" @@ -404,6 +405,7 @@ channel_register(channel_t *chan) /* Put it in the finished list, creating it if necessary */ if (!finished_channels) finished_channels = smartlist_new(); smartlist_add(finished_channels, chan); + mainloop_schedule_postloop_cleanup(); } else { /* Put it in the active list, creating it if necessary */ if (!active_channels) active_channels = smartlist_new(); @@ -1548,6 +1550,7 @@ channel_change_state_(channel_t *chan, channel_state_t to_state) if (active_channels) smartlist_remove(active_channels, chan); if (!finished_channels) finished_channels = smartlist_new(); smartlist_add(finished_channels, chan); + mainloop_schedule_postloop_cleanup(); } /* Need to put on active list? */ else if (!was_active && is_active) { @@ -1666,6 +1669,7 @@ channel_listener_change_state(channel_listener_t *chan_l, if (active_listeners) smartlist_remove(active_listeners, chan_l); if (!finished_listeners) finished_listeners = smartlist_new(); smartlist_add(finished_listeners, chan_l); + mainloop_schedule_postloop_cleanup(); } /* Need to put on active list? */ else if (!was_active && is_active) { @@ -2109,21 +2113,6 @@ channel_listener_dumpstats(int severity) } /** - * Set the cmux policy on all active channels. - */ -void -channel_set_cmux_policy_everywhere(circuitmux_policy_t *pol) -{ - if (!active_channels) return; - - SMARTLIST_FOREACH_BEGIN(active_channels, channel_t *, curr) { - if (curr->cmux) { - circuitmux_set_policy(curr->cmux, pol); - } - } SMARTLIST_FOREACH_END(curr); -} - -/** * Clean up channels. * * This gets called periodically from run_scheduled_events() in main.c; @@ -2402,7 +2391,7 @@ channel_get_for_extend(const char *rsa_id_digest, { channel_t *chan, *best = NULL; int n_inprogress_goodaddr = 0, n_old = 0; - int n_noncanonical = 0, n_possible = 0; + int n_noncanonical = 0; tor_assert(msg_out); tor_assert(launch_out); @@ -2465,8 +2454,6 @@ channel_get_for_extend(const char *rsa_id_digest, continue; } - ++n_possible; - if (!best) { best = chan; /* If we have no 'best' so far, this one is good enough. */ continue; diff --git a/src/or/channel.h b/src/or/channel.h index 0af5aed414..6cf8cd7f72 100644 --- a/src/or/channel.h +++ b/src/or/channel.h @@ -422,9 +422,6 @@ void channel_free_all(void); void channel_dumpstats(int severity); void channel_listener_dumpstats(int severity); -/* Set the cmux policy on all active channels */ -void channel_set_cmux_policy_everywhere(circuitmux_policy_t *pol); - #ifdef TOR_CHANNEL_INTERNAL_ #ifdef CHANNEL_PRIVATE_ diff --git a/src/or/channelpadding.c b/src/or/channelpadding.c index ba432e716b..3dbdd019ab 100644 --- a/src/or/channelpadding.c +++ b/src/or/channelpadding.c @@ -16,11 +16,11 @@ #include "networkstatus.h" #include "connection.h" #include "connection_or.h" +#include "crypto_rand.h" #include "main.h" #include "rephist.h" #include "router.h" #include "compat_time.h" -#include <event2/event.h> #include "rendservice.h" STATIC int32_t channelpadding_get_netflow_inactive_timeout_ms( diff --git a/src/or/channeltls.c b/src/or/channeltls.c index 9000703b01..54d94f6109 100644 --- a/src/or/channeltls.c +++ b/src/or/channeltls.c @@ -160,9 +160,8 @@ channel_tls_common_init(channel_tls_t *tlschan) chan->write_var_cell = channel_tls_write_var_cell_method; chan->cmux = circuitmux_alloc(); - if (cell_ewma_enabled()) { - circuitmux_set_policy(chan->cmux, &ewma_policy); - } + /* We only have one policy for now so always set it to EWMA. */ + circuitmux_set_policy(chan->cmux, &ewma_policy); } /** diff --git a/src/or/circpathbias.c b/src/or/circpathbias.c index c1c1ca31be..ff42bf91e4 100644 --- a/src/or/circpathbias.c +++ b/src/or/circpathbias.c @@ -30,6 +30,7 @@ #include "circuitstats.h" #include "connection_edge.h" #include "config.h" +#include "crypto_rand.h" #include "entrynodes.h" #include "networkstatus.h" #include "relay.h" diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index 75307c367f..3d1c9c1abf 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -43,7 +43,7 @@ #include "connection_edge.h" #include "connection_or.h" #include "control.h" -#include "crypto.h" +#include "crypto_rand.h" #include "directory.h" #include "entrynodes.h" #include "hs_ntor.h" @@ -56,6 +56,7 @@ #include "onion_fast.h" #include "policies.h" #include "relay.h" +#include "relay_crypto.h" #include "rendcommon.h" #include "rephist.h" #include "router.h" @@ -71,10 +72,7 @@ static channel_t * channel_connect_for_circuit(const tor_addr_t *addr, static int circuit_deliver_create_cell(circuit_t *circ, const create_cell_t *create_cell, int relayed); -static int onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit, - int is_hs_v3_rp_circuit); static crypt_path_t *onion_next_hop_in_cpath(crypt_path_t *cpath); -static int onion_extend_cpath(origin_circuit_t *circ); STATIC int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice); static int circuit_send_first_onion_skin(origin_circuit_t *circ); static int circuit_build_no_more_hops(origin_circuit_t *circ); @@ -1055,7 +1053,7 @@ circuit_build_no_more_hops(origin_circuit_t *circ) clear_broken_connection_map(1); if (server_mode(options) && !check_whether_orport_reachable(options)) { inform_testing_reachability(); - consider_testing_reachability(1, 1); + router_do_reachability_checks(1, 1); } } @@ -1132,19 +1130,29 @@ circuit_send_intermediate_onion_skin(origin_circuit_t *circ, return 0; } -/** Our clock just jumped by <b>seconds_elapsed</b>. Assume - * something has also gone wrong with our network: notify the user, - * and abandon all not-yet-used circuits. */ +/** Our clock just jumped by <b>seconds_elapsed</b>. If <b>was_idle</b> is + * true, then the monotonic time matches; otherwise it doesn't. Assume + * something has also gone wrong with our network: notify the user, and + * abandon all not-yet-used circuits. */ void -circuit_note_clock_jumped(int seconds_elapsed) +circuit_note_clock_jumped(int64_t seconds_elapsed, bool was_idle) { int severity = server_mode(get_options()) ? LOG_WARN : LOG_NOTICE; - tor_log(severity, LD_GENERAL, "Your system clock just jumped %d seconds %s; " - "assuming established circuits no longer work.", - seconds_elapsed >=0 ? seconds_elapsed : -seconds_elapsed, - seconds_elapsed >=0 ? "forward" : "backward"); - control_event_general_status(LOG_WARN, "CLOCK_JUMPED TIME=%d", - seconds_elapsed); + if (was_idle) { + tor_log(severity, LD_GENERAL, "Tor has been idle for "I64_FORMAT + " seconds; assuming established circuits no longer work.", + I64_PRINTF_ARG(seconds_elapsed)); + } else { + tor_log(severity, LD_GENERAL, + "Your system clock just jumped "I64_FORMAT" seconds %s; " + "assuming established circuits no longer work.", + I64_PRINTF_ARG( + seconds_elapsed >=0 ? seconds_elapsed : -seconds_elapsed), + seconds_elapsed >=0 ? "forward" : "backward"); + } + control_event_general_status(LOG_WARN, "CLOCK_JUMPED TIME="I64_FORMAT + " IDLE=%d", + I64_PRINTF_ARG(seconds_elapsed), was_idle?1:0); /* so we log when it works again */ note_that_we_maybe_cant_complete_circuits(); control_event_client_status(severity, "CIRCUIT_NOT_ESTABLISHED REASON=%s", @@ -1337,69 +1345,10 @@ circuit_init_cpath_crypto(crypt_path_t *cpath, const char *key_data, size_t key_data_len, int reverse, int is_hs_v3) { - crypto_digest_t *tmp_digest; - crypto_cipher_t *tmp_crypto; - size_t digest_len = 0; - size_t cipher_key_len = 0; tor_assert(cpath); - tor_assert(key_data); - tor_assert(!(cpath->f_crypto || cpath->b_crypto || - cpath->f_digest || cpath->b_digest)); - - /* Basic key size validation */ - if (is_hs_v3 && BUG(key_data_len != HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN)) { - return -1; - } else if (!is_hs_v3 && BUG(key_data_len != CPATH_KEY_MATERIAL_LEN)) { - return -1; - } - - /* If we are using this cpath for next gen onion services use SHA3-256, - otherwise use good ol' SHA1 */ - if (is_hs_v3) { - digest_len = DIGEST256_LEN; - cipher_key_len = CIPHER256_KEY_LEN; - cpath->f_digest = crypto_digest256_new(DIGEST_SHA3_256); - cpath->b_digest = crypto_digest256_new(DIGEST_SHA3_256); - } else { - digest_len = DIGEST_LEN; - cipher_key_len = CIPHER_KEY_LEN; - cpath->f_digest = crypto_digest_new(); - cpath->b_digest = crypto_digest_new(); - } - - tor_assert(digest_len != 0); - tor_assert(cipher_key_len != 0); - const int cipher_key_bits = (int) cipher_key_len * 8; - - crypto_digest_add_bytes(cpath->f_digest, key_data, digest_len); - crypto_digest_add_bytes(cpath->b_digest, key_data+digest_len, digest_len); - - cpath->f_crypto = crypto_cipher_new_with_bits(key_data+(2*digest_len), - cipher_key_bits); - if (!cpath->f_crypto) { - log_warn(LD_BUG,"Forward cipher initialization failed."); - return -1; - } - - cpath->b_crypto = crypto_cipher_new_with_bits( - key_data+(2*digest_len)+cipher_key_len, - cipher_key_bits); - if (!cpath->b_crypto) { - log_warn(LD_BUG,"Backward cipher initialization failed."); - return -1; - } - - if (reverse) { - tmp_digest = cpath->f_digest; - cpath->f_digest = cpath->b_digest; - cpath->b_digest = tmp_digest; - tmp_crypto = cpath->f_crypto; - cpath->f_crypto = cpath->b_crypto; - cpath->b_crypto = tmp_crypto; - } - - return 0; + return relay_crypto_init(&cpath->crypto, key_data, key_data_len, reverse, + is_hs_v3); } /** A "created" cell <b>reply</b> came back to us on circuit <b>circ</b>. @@ -1522,7 +1471,6 @@ onionskin_answer(or_circuit_t *circ, const uint8_t *rend_circ_nonce) { cell_t cell; - crypt_path_t *tmp_cpath; tor_assert(keys_len == CPATH_KEY_MATERIAL_LEN); @@ -1533,25 +1481,15 @@ onionskin_answer(or_circuit_t *circ, } cell.circ_id = circ->p_circ_id; - tmp_cpath = tor_malloc_zero(sizeof(crypt_path_t)); - tmp_cpath->magic = CRYPT_PATH_MAGIC; - circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_OPEN); log_debug(LD_CIRC,"init digest forward 0x%.8x, backward 0x%.8x.", (unsigned int)get_uint32(keys), (unsigned int)get_uint32(keys+20)); - if (circuit_init_cpath_crypto(tmp_cpath, keys, keys_len, 0, 0)<0) { + if (relay_crypto_init(&circ->crypto, keys, keys_len, 0, 0)<0) { log_warn(LD_BUG,"Circuit initialization failed"); - tor_free(tmp_cpath); return -1; } - circ->n_digest = tmp_cpath->f_digest; - circ->n_crypto = tmp_cpath->f_crypto; - circ->p_digest = tmp_cpath->b_digest; - circ->p_crypto = tmp_cpath->b_crypto; - tmp_cpath->magic = 0; - tor_free(tmp_cpath); memcpy(circ->rend_circ_nonce, rend_circ_nonce, DIGEST_LEN); @@ -1612,7 +1550,7 @@ onionskin_answer(or_circuit_t *circ, * rend_service_launch_establish_intro()) * * - We are a router testing its own reachabiity - * (CIRCUIT_PURPOSE_TESTING, via consider_testing_reachability()) + * (CIRCUIT_PURPOSE_TESTING, via router_do_reachability_checks()) * * onion_pick_cpath_exit() bypasses us (by not calling * new_route_len()) in the one-hop tunnel case, so we don't need to @@ -2353,7 +2291,7 @@ warn_if_last_router_excluded(origin_circuit_t *circ, * be used as an HS v3 rendezvous point. * * Return 0 if ok, -1 if circuit should be closed. */ -static int +STATIC int onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit_ei, int is_hs_v3_rp_circuit) { @@ -2524,12 +2462,71 @@ cpath_get_n_hops(crypt_path_t **head_ptr) #endif /* defined(TOR_UNIT_TESTS) */ /** + * Build the exclude list for vanguard circuits. + * + * For vanguard circuits we exclude all the already chosen nodes (including the + * exit) from being middle hops to prevent the creation of A - B - A subpaths. + * We also allow the 4th hop to be the same as the guard node so as to not leak + * guard information to RP/IP/HSDirs. + * + * For vanguard circuits, we don't apply any subnet or family restrictions. + * This is to avoid impossible-to-build circuit paths, or just situations where + * our earlier guards prevent us from using most of our later ones. + * + * The alternative is building the circuit in reverse. Reverse calls to + * onion_extend_cpath() (ie: select outer hops first) would then have the + * property that you don't gain information about inner hops by observing + * outer ones. See https://trac.torproject.org/projects/tor/ticket/24487 + * for this. + * + * (Note further that we still exclude the exit to prevent A - B - A + * at the end of the path. */ +static smartlist_t * +build_vanguard_middle_exclude_list(uint8_t purpose, + cpath_build_state_t *state, + crypt_path_t *head, + int cur_len) +{ + smartlist_t *excluded; + const node_t *r; + crypt_path_t *cpath; + int i; + + (void) purpose; + + excluded = smartlist_new(); + + /* Add the exit to the exclude list (note that the exit/last hop is always + * chosen first in circuit_establish_circuit()). */ + if ((r = build_state_get_exit_node(state))) { + smartlist_add(excluded, (node_t*)r); + } + + /* If we are picking the 4th hop, allow that node to be the guard too. + * This prevents us from avoiding the Guard for those hops, which + * gives the adversary information about our guard if they control + * the RP, IP, or HSDIR. We don't do this check based on purpose + * because we also want to allow HS_VANGUARDS pre-build circuits + * to use the guard for that last hop. + */ + if (cur_len == DEFAULT_ROUTE_LEN+1) { + /* Skip the first hop for the exclude list below */ + head = head->next; + cur_len--; + } + + for (i = 0, cpath = head; cpath && i < cur_len; ++i, cpath=cpath->next) { + if ((r = node_get_by_id(cpath->extend_info->identity_digest))) { + smartlist_add(excluded, (node_t*)r); + } + } + + return excluded; +} + +/** * Build a list of nodes to exclude from the choice of this middle * hop, based on already chosen nodes. - * - * XXX: At present, this function does not exclude any nodes from - * the vanguard circuits. See - * https://trac.torproject.org/projects/tor/ticket/24487 */ static smartlist_t * build_middle_exclude_list(uint8_t purpose, @@ -2542,32 +2539,21 @@ build_middle_exclude_list(uint8_t purpose, crypt_path_t *cpath; int i; + /** Vanguard circuits have their own path selection rules */ + if (circuit_should_use_vanguards(purpose)) { + return build_vanguard_middle_exclude_list(purpose, state, head, cur_len); + } + excluded = smartlist_new(); - /* Add the exit to the exclude list (note that the exit/last hop is always - * chosen first in circuit_establish_circuit()). */ + /* For non-vanguard circuits, add the exit and its family to the exclude list + * (note that the exit/last hop is always chosen first in + * circuit_establish_circuit()). */ if ((r = build_state_get_exit_node(state))) { nodelist_add_node_and_family(excluded, r); } - /* XXX: We don't apply any other previously selected node restrictions for - * vanguards, and allow nodes to be reused for those hop positions in the - * same circuit. This is because after many rotations, you get to learn - * inner guard nodes through the nodes that are not selected for outer - * hops. - * - * The alternative is building the circuit in reverse. Reverse calls to - * onion_extend_cpath() (ie: select outer hops first) would then have the - * property that you don't gain information about inner hops by observing - * outer ones. See https://trac.torproject.org/projects/tor/ticket/24487 - * for this. - * - * (Note further that we can and do still exclude the exit in the block - * above, because it is chosen first in circuit_establish_circuit()..) */ - if (circuit_should_use_vanguards(purpose)) { - return excluded; - } - + /* also exclude all other already chosen nodes and their family */ for (i = 0, cpath = head; cpath && i < cur_len; ++i, cpath=cpath->next) { if ((r = node_get_by_id(cpath->extend_info->identity_digest))) { nodelist_add_node_and_family(excluded, r); @@ -2667,7 +2653,9 @@ choose_good_middle_server(uint8_t purpose, /** If a hidden service circuit wants a specific middle node, pin it. */ if (middle_node_must_be_vanguard(options, purpose, cur_len)) { log_debug(LD_GENERAL, "Picking a sticky node (cur_len = %d)", cur_len); - return pick_vanguard_middle_node(options, flags, cur_len, excluded); + choice = pick_vanguard_middle_node(options, flags, cur_len, excluded); + smartlist_free(excluded); + return choice; } choice = router_choose_random_node(excluded, options->ExcludeNodes, flags); @@ -2707,7 +2695,7 @@ choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state, /* This request is for an entry server to use for a regular circuit, * and we use entry guard nodes. Just return one of the guard nodes. */ tor_assert(guard_state_out); - return guards_choose_guard(state, guard_state_out); + return guards_choose_guard(state, purpose, guard_state_out); } excluded = smartlist_new(); @@ -2750,7 +2738,7 @@ onion_next_hop_in_cpath(crypt_path_t *cpath) * Return 1 if the path is complete, 0 if we successfully added a hop, * and -1 on error. */ -static int +STATIC int onion_extend_cpath(origin_circuit_t *circ) { uint8_t purpose = circ->base_.purpose; @@ -2872,14 +2860,13 @@ extend_info_from_node(const node_t *node, int for_direct_connect) return NULL; } - /* Choose a preferred address first, but fall back to an allowed address. - * choose_address returns 1 on success, but get_prim_orport returns 0. */ + /* Choose a preferred address first, but fall back to an allowed address. */ if (for_direct_connect) - valid_addr = fascist_firewall_choose_address_node(node, - FIREWALL_OR_CONNECTION, - 0, &ap); - else - valid_addr = !node_get_prim_orport(node, &ap); + fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0, &ap); + else { + node_get_prim_orport(node, &ap); + } + valid_addr = tor_addr_port_is_valid_ap(&ap, 0); if (valid_addr) log_debug(LD_CIRC, "using %s for %s", diff --git a/src/or/circuitbuild.h b/src/or/circuitbuild.h index bea31ad0dd..0184898e29 100644 --- a/src/or/circuitbuild.h +++ b/src/or/circuitbuild.h @@ -29,7 +29,7 @@ void circuit_n_chan_done(channel_t *chan, int status, int inform_testing_reachability(void); int circuit_timeout_want_to_count_circ(const origin_circuit_t *circ); int circuit_send_next_onion_skin(origin_circuit_t *circ); -void circuit_note_clock_jumped(int seconds_elapsed); +void circuit_note_clock_jumped(int64_t seconds_elapsed, bool was_idle); int circuit_extend(cell_t *cell, circuit_t *circ); int circuit_init_cpath_crypto(crypt_path_t *cpath, const char *key_data, size_t key_data_len, @@ -83,6 +83,13 @@ STATIC circid_t get_unique_circ_id_by_chan(channel_t *chan); STATIC int new_route_len(uint8_t purpose, extend_info_t *exit_ei, smartlist_t *nodes); MOCK_DECL(STATIC int, count_acceptable_nodes, (smartlist_t *nodes)); + +STATIC int onion_extend_cpath(origin_circuit_t *circ); + +STATIC int +onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit_ei, + int is_hs_v3_rp_circuit); + #if defined(ENABLE_TOR2WEB_MODE) || defined(TOR_UNIT_TESTS) STATIC const node_t *pick_tor2web_rendezvous_node(router_crn_flags_t flags, const or_options_t *options); diff --git a/src/or/circuitlist.c b/src/or/circuitlist.c index 7bdef0b878..45fff7cc17 100644 --- a/src/or/circuitlist.c +++ b/src/or/circuitlist.c @@ -65,6 +65,8 @@ #include "connection_edge.h" #include "connection_or.h" #include "control.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "entrynodes.h" #include "main.h" #include "hs_circuit.h" @@ -76,6 +78,7 @@ #include "onion_fast.h" #include "policies.h" #include "relay.h" +#include "relay_crypto.h" #include "rendclient.h" #include "rendcommon.h" #include "rephist.h" @@ -406,9 +409,6 @@ circuit_set_p_circid_chan(or_circuit_t *or_circ, circid_t id, circuit_set_circid_chan_helper(circ, CELL_DIRECTION_IN, id, chan); if (chan) { - tor_assert(bool_eq(or_circ->p_chan_cells.n, - or_circ->next_active_on_p_chan)); - chan->timestamp_last_had_circuits = approx_time(); } @@ -431,8 +431,6 @@ circuit_set_n_circid_chan(circuit_t *circ, circid_t id, circuit_set_circid_chan_helper(circ, CELL_DIRECTION_OUT, id, chan); if (chan) { - tor_assert(bool_eq(circ->n_chan_cells.n, circ->next_active_on_n_chan)); - chan->timestamp_last_had_circuits = approx_time(); } @@ -1087,10 +1085,7 @@ circuit_free_(circuit_t *circ) should_free = (ocirc->workqueue_entry == NULL); - crypto_cipher_free(ocirc->p_crypto); - crypto_digest_free(ocirc->p_digest); - crypto_cipher_free(ocirc->n_crypto); - crypto_digest_free(ocirc->n_digest); + relay_crypto_clear(ô->crypto); if (ocirc->rend_splice) { or_circuit_t *other = ocirc->rend_splice; @@ -1230,10 +1225,7 @@ circuit_free_cpath_node(crypt_path_t *victim) if (!victim) return; - crypto_cipher_free(victim->f_crypto); - crypto_cipher_free(victim->b_crypto); - crypto_digest_free(victim->f_digest); - crypto_digest_free(victim->b_digest); + relay_crypto_clear(&victim->crypto); onion_handshake_state_release(&victim->handshake_state); crypto_dh_free(victim->rend_dh_handshake_state); extend_info_free(victim->extend_info); @@ -2077,6 +2069,7 @@ circuit_mark_for_close_, (circuit_t *circ, int reason, int line, circuits_pending_close = smartlist_new(); smartlist_add(circuits_pending_close, circ); + mainloop_schedule_postloop_cleanup(); log_info(LD_GENERAL, "Circuit %u (id: %" PRIu32 ") marked for close at " "%s:%d (orig reason: %d, new reason: %d)", @@ -2596,8 +2589,7 @@ assert_cpath_layer_ok(const crypt_path_t *cp) switch (cp->state) { case CPATH_STATE_OPEN: - tor_assert(cp->f_crypto); - tor_assert(cp->b_crypto); + relay_crypto_assert_ok(&cp->crypto); /* fall through */ case CPATH_STATE_CLOSED: /*XXXX Assert that there's no handshake_state either. */ @@ -2687,10 +2679,7 @@ assert_circuit_ok,(const circuit_t *c)) c->state == CIRCUIT_STATE_GUARD_WAIT) { tor_assert(!c->n_chan_create_cell); if (or_circ) { - tor_assert(or_circ->n_crypto); - tor_assert(or_circ->p_crypto); - tor_assert(or_circ->n_digest); - tor_assert(or_circ->p_digest); + relay_crypto_assert_ok(&or_circ->crypto); } } if (c->state == CIRCUIT_STATE_CHAN_WAIT && !c->marked_for_close) { diff --git a/src/or/circuitmux.c b/src/or/circuitmux.c index fe3d8f1332..f9f5faa057 100644 --- a/src/or/circuitmux.c +++ b/src/or/circuitmux.c @@ -114,13 +114,6 @@ struct circuitmux_s { */ chanid_circid_muxinfo_map_t *chanid_circid_map; - /* - * Double-linked ring of circuits with queued cells waiting for room to - * free up on this connection's outbuf. Every time we pull cells from - * a circuit, we advance this pointer to the next circuit in the ring. - */ - struct circuit_t *active_circuits_head, *active_circuits_tail; - /** List of queued destroy cells */ destroy_cell_queue_t destroy_cell_queue; /** Boolean: True iff the last cell to circuitmux_get_first_active_circuit @@ -177,17 +170,6 @@ struct chanid_circid_muxinfo_t { }; /* - * Internal-use #defines - */ - -#ifdef CMUX_PARANOIA -#define circuitmux_assert_okay_paranoid(cmux) \ - circuitmux_assert_okay(cmux) -#else -#define circuitmux_assert_okay_paranoid(cmux) -#endif /* defined(CMUX_PARANOIA) */ - -/* * Static function declarations */ @@ -199,21 +181,9 @@ chanid_circid_entry_hash(chanid_circid_muxinfo_t *a); static chanid_circid_muxinfo_t * circuitmux_find_map_entry(circuitmux_t *cmux, circuit_t *circ); static void -circuitmux_make_circuit_active(circuitmux_t *cmux, circuit_t *circ, - cell_direction_t direction); +circuitmux_make_circuit_active(circuitmux_t *cmux, circuit_t *circ); static void -circuitmux_make_circuit_inactive(circuitmux_t *cmux, circuit_t *circ, - cell_direction_t direction); -static inline void -circuitmux_move_active_circ_to_tail(circuitmux_t *cmux, circuit_t *circ, - cell_direction_t direction); -static inline circuit_t ** -circuitmux_next_active_circ_p(circuitmux_t *cmux, circuit_t *circ); -static inline circuit_t ** -circuitmux_prev_active_circ_p(circuitmux_t *cmux, circuit_t *circ); -static void circuitmux_assert_okay_pass_one(circuitmux_t *cmux); -static void circuitmux_assert_okay_pass_two(circuitmux_t *cmux); -static void circuitmux_assert_okay_pass_three(circuitmux_t *cmux); +circuitmux_make_circuit_inactive(circuitmux_t *cmux, circuit_t *circ); /* Static global variables */ @@ -223,119 +193,6 @@ static int64_t global_destroy_ctr = 0; /* Function definitions */ /** - * Linked list helpers - */ - -/** - * Move an active circuit to the tail of the cmux's active circuits list; - * used by circuitmux_notify_xmit_cells(). - */ - -static inline void -circuitmux_move_active_circ_to_tail(circuitmux_t *cmux, circuit_t *circ, - cell_direction_t direction) -{ - circuit_t **next_p = NULL, **prev_p = NULL; - circuit_t **next_prev = NULL, **prev_next = NULL; - circuit_t **tail_next = NULL; - or_circuit_t *or_circ = NULL; - - tor_assert(cmux); - tor_assert(circ); - - circuitmux_assert_okay_paranoid(cmux); - - /* Figure out our next_p and prev_p for this cmux/direction */ - if (direction) { - if (direction == CELL_DIRECTION_OUT) { - tor_assert(circ->n_mux == cmux); - next_p = &(circ->next_active_on_n_chan); - prev_p = &(circ->prev_active_on_n_chan); - } else { - or_circ = TO_OR_CIRCUIT(circ); - tor_assert(or_circ->p_mux == cmux); - next_p = &(or_circ->next_active_on_p_chan); - prev_p = &(or_circ->prev_active_on_p_chan); - } - } else { - if (circ->n_mux == cmux) { - next_p = &(circ->next_active_on_n_chan); - prev_p = &(circ->prev_active_on_n_chan); - } else { - or_circ = TO_OR_CIRCUIT(circ); - tor_assert(or_circ->p_mux == cmux); - next_p = &(or_circ->next_active_on_p_chan); - prev_p = &(or_circ->prev_active_on_p_chan); - } - } - tor_assert(next_p); - tor_assert(prev_p); - - /* Check if this really is an active circuit */ - if ((*next_p == NULL && *prev_p == NULL) && - !(circ == cmux->active_circuits_head || - circ == cmux->active_circuits_tail)) { - /* Not active, no-op */ - return; - } - - /* Check if this is already the tail */ - if (circ == cmux->active_circuits_tail) return; - - /* Okay, we have to move it; figure out next_prev and prev_next */ - if (*next_p) next_prev = circuitmux_prev_active_circ_p(cmux, *next_p); - if (*prev_p) prev_next = circuitmux_next_active_circ_p(cmux, *prev_p); - /* Adjust the previous node's next pointer, if any */ - if (prev_next) *prev_next = *next_p; - /* Otherwise, we were the head */ - else cmux->active_circuits_head = *next_p; - /* Adjust the next node's previous pointer, if any */ - if (next_prev) *next_prev = *prev_p; - /* We're out of the list; now re-attach at the tail */ - /* Adjust our next and prev pointers */ - *next_p = NULL; - *prev_p = cmux->active_circuits_tail; - /* Set the next pointer of the tail, or the head if none */ - if (cmux->active_circuits_tail) { - tail_next = circuitmux_next_active_circ_p(cmux, - cmux->active_circuits_tail); - *tail_next = circ; - } else { - cmux->active_circuits_head = circ; - } - /* Set the tail to this circuit */ - cmux->active_circuits_tail = circ; - - circuitmux_assert_okay_paranoid(cmux); -} - -static inline circuit_t ** -circuitmux_next_active_circ_p(circuitmux_t *cmux, circuit_t *circ) -{ - tor_assert(cmux); - tor_assert(circ); - - if (circ->n_mux == cmux) return &(circ->next_active_on_n_chan); - else { - tor_assert(TO_OR_CIRCUIT(circ)->p_mux == cmux); - return &(TO_OR_CIRCUIT(circ)->next_active_on_p_chan); - } -} - -static inline circuit_t ** -circuitmux_prev_active_circ_p(circuitmux_t *cmux, circuit_t *circ) -{ - tor_assert(cmux); - tor_assert(circ); - - if (circ->n_mux == cmux) return &(circ->prev_active_on_n_chan); - else { - tor_assert(TO_OR_CIRCUIT(circ)->p_mux == cmux); - return &(TO_OR_CIRCUIT(circ)->prev_active_on_p_chan); - } -} - -/** * Helper for chanid_circid_cell_count_map_t hash table: compare the channel * ID and circuit ID for a and b, and return less than, equal to, or greater * than zero appropriately. @@ -406,11 +263,6 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux, smartlist_t *detached_out) circuit_t *circ = NULL; tor_assert(cmux); - /* - * Don't circuitmux_assert_okay_paranoid() here; this gets called when - * channels are being freed and have already been unregistered, so - * the channel ID lookups it does will fail. - */ i = HT_START(chanid_circid_muxinfo_map, cmux->chanid_circid_map); while (i) { @@ -435,7 +287,7 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux, smartlist_t *detached_out) */ if (to_remove->muxinfo.cell_count > 0) { - circuitmux_make_circuit_inactive(cmux, circ, CELL_DIRECTION_OUT); + circuitmux_make_circuit_inactive(cmux, circ); } /* Clear n_mux */ @@ -450,7 +302,7 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux, smartlist_t *detached_out) */ if (to_remove->muxinfo.cell_count > 0) { - circuitmux_make_circuit_inactive(cmux, circ, CELL_DIRECTION_IN); + circuitmux_make_circuit_inactive(cmux, circ); } /* @@ -606,9 +458,7 @@ circuitmux_clear_policy(circuitmux_t *cmux) tor_assert(cmux); /* Internally, this is just setting policy to NULL */ - if (cmux->policy) { - circuitmux_set_policy(cmux, NULL); - } + circuitmux_set_policy(cmux, NULL); } /** @@ -944,7 +794,6 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ, tor_assert(circ); tor_assert(direction == CELL_DIRECTION_IN || direction == CELL_DIRECTION_OUT); - circuitmux_assert_okay_paranoid(cmux); /* * Figure out which channel we're using, and get the circuit's current @@ -1002,10 +851,10 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ, */ if (hashent->muxinfo.cell_count > 0 && cell_count == 0) { --(cmux->n_active_circuits); - circuitmux_make_circuit_inactive(cmux, circ, direction); + circuitmux_make_circuit_inactive(cmux, circ); } else if (hashent->muxinfo.cell_count == 0 && cell_count > 0) { ++(cmux->n_active_circuits); - circuitmux_make_circuit_active(cmux, circ, direction); + circuitmux_make_circuit_active(cmux, circ); } cmux->n_cells -= hashent->muxinfo.cell_count; cmux->n_cells += cell_count; @@ -1033,7 +882,7 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ, hashent->muxinfo.cell_count = cell_count; hashent->muxinfo.direction = direction; /* Allocate policy specific circuit data if we need it */ - if (cmux->policy && cmux->policy->alloc_circ_data) { + if (cmux->policy->alloc_circ_data) { /* Assert that we have the means to free policy-specific data */ tor_assert(cmux->policy->free_circ_data); /* Allocate it */ @@ -1053,25 +902,14 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ, if (direction == CELL_DIRECTION_OUT) circ->n_mux = cmux; else TO_OR_CIRCUIT(circ)->p_mux = cmux; - /* Make sure the next/prev pointers are NULL */ - if (direction == CELL_DIRECTION_OUT) { - circ->next_active_on_n_chan = NULL; - circ->prev_active_on_n_chan = NULL; - } else { - TO_OR_CIRCUIT(circ)->next_active_on_p_chan = NULL; - TO_OR_CIRCUIT(circ)->prev_active_on_p_chan = NULL; - } - /* Update counters */ ++(cmux->n_circuits); if (cell_count > 0) { ++(cmux->n_active_circuits); - circuitmux_make_circuit_active(cmux, circ, direction); + circuitmux_make_circuit_active(cmux, circ); } cmux->n_cells += cell_count; } - - circuitmux_assert_okay_paranoid(cmux); } /** @@ -1095,7 +933,6 @@ circuitmux_detach_circuit,(circuitmux_t *cmux, circuit_t *circ)) tor_assert(cmux); tor_assert(cmux->chanid_circid_map); tor_assert(circ); - circuitmux_assert_okay_paranoid(cmux); /* See if we have it for n_chan/n_circ_id */ if (circ->n_chan) { @@ -1133,7 +970,7 @@ circuitmux_detach_circuit,(circuitmux_t *cmux, circuit_t *circ)) if (hashent->muxinfo.cell_count > 0) { --(cmux->n_active_circuits); /* This does policy notifies, so comes before freeing policy data */ - circuitmux_make_circuit_inactive(cmux, circ, last_searched_direction); + circuitmux_make_circuit_inactive(cmux, circ); } cmux->n_cells -= hashent->muxinfo.cell_count; @@ -1162,8 +999,6 @@ circuitmux_detach_circuit,(circuitmux_t *cmux, circuit_t *circ)) /* Free the hash entry */ tor_free(hashent); } - - circuitmux_assert_okay_paranoid(cmux); } /** @@ -1172,94 +1007,22 @@ circuitmux_detach_circuit,(circuitmux_t *cmux, circuit_t *circ)) */ static void -circuitmux_make_circuit_active(circuitmux_t *cmux, circuit_t *circ, - cell_direction_t direction) +circuitmux_make_circuit_active(circuitmux_t *cmux, circuit_t *circ) { - circuit_t **next_active = NULL, **prev_active = NULL, **next_prev = NULL; - circuitmux_t *circuit_cmux = NULL; - chanid_circid_muxinfo_t *hashent = NULL; - channel_t *chan = NULL; - circid_t circ_id; - int already_active; - tor_assert(cmux); + tor_assert(cmux->policy); tor_assert(circ); - tor_assert(direction == CELL_DIRECTION_OUT || - direction == CELL_DIRECTION_IN); - /* - * Don't circuitmux_assert_okay_paranoid(cmux) here because the cell count - * already got changed and we have to update the list for it to be consistent - * again. - */ - - /* Get the right set of active list links for this direction */ - if (direction == CELL_DIRECTION_OUT) { - next_active = &(circ->next_active_on_n_chan); - prev_active = &(circ->prev_active_on_n_chan); - circuit_cmux = circ->n_mux; - chan = circ->n_chan; - circ_id = circ->n_circ_id; - } else { - next_active = &(TO_OR_CIRCUIT(circ)->next_active_on_p_chan); - prev_active = &(TO_OR_CIRCUIT(circ)->prev_active_on_p_chan); - circuit_cmux = TO_OR_CIRCUIT(circ)->p_mux; - chan = TO_OR_CIRCUIT(circ)->p_chan; - circ_id = TO_OR_CIRCUIT(circ)->p_circ_id; - } - - /* Assert that it is attached to this mux and a channel */ - tor_assert(cmux == circuit_cmux); - tor_assert(chan != NULL); - - /* - * Check if the circuit really was inactive; if it's active, at least one - * of the next_active and prev_active pointers will not be NULL, or this - * circuit will be either the head or tail of the list for this cmux. - */ - already_active = (*prev_active != NULL || *next_active != NULL || - cmux->active_circuits_head == circ || - cmux->active_circuits_tail == circ); - - /* If we're already active, log a warning and finish */ - if (already_active) { - log_warn(LD_CIRC, - "Circuit %u on channel " U64_FORMAT " was already active", - (unsigned)circ_id, U64_PRINTF_ARG(chan->global_identifier)); - return; - } - - /* - * This is going at the head of the list; if the old head is not NULL, - * then its prev pointer should point to this. - */ - *next_active = cmux->active_circuits_head; /* Next is old head */ - *prev_active = NULL; /* Prev is NULL (this will be the head) */ - if (cmux->active_circuits_head) { - /* The list had an old head; update its prev pointer */ - next_prev = - circuitmux_prev_active_circ_p(cmux, cmux->active_circuits_head); - tor_assert(next_prev); - *next_prev = circ; - } else { - /* The list was empty; this becomes the tail as well */ - cmux->active_circuits_tail = circ; - } - /* This becomes the new head of the list */ - cmux->active_circuits_head = circ; /* Policy-specific notification */ - if (cmux->policy && - cmux->policy->notify_circ_active) { + if (cmux->policy->notify_circ_active) { /* Okay, we need to check the circuit for policy data now */ - hashent = circuitmux_find_map_entry(cmux, circ); + chanid_circid_muxinfo_t *hashent = circuitmux_find_map_entry(cmux, circ); /* We should have found something */ tor_assert(hashent); /* Notify */ cmux->policy->notify_circ_active(cmux, cmux->policy_data, circ, hashent->muxinfo.policy_data); } - - circuitmux_assert_okay_paranoid(cmux); } /** @@ -1268,112 +1031,22 @@ circuitmux_make_circuit_active(circuitmux_t *cmux, circuit_t *circ, */ static void -circuitmux_make_circuit_inactive(circuitmux_t *cmux, circuit_t *circ, - cell_direction_t direction) +circuitmux_make_circuit_inactive(circuitmux_t *cmux, circuit_t *circ) { - circuit_t **next_active = NULL, **prev_active = NULL; - circuit_t **next_prev = NULL, **prev_next = NULL; - circuitmux_t *circuit_cmux = NULL; - chanid_circid_muxinfo_t *hashent = NULL; - channel_t *chan = NULL; - circid_t circ_id; - int already_inactive; - tor_assert(cmux); + tor_assert(cmux->policy); tor_assert(circ); - tor_assert(direction == CELL_DIRECTION_OUT || - direction == CELL_DIRECTION_IN); - /* - * Don't circuitmux_assert_okay_paranoid(cmux) here because the cell count - * already got changed and we have to update the list for it to be consistent - * again. - */ - - /* Get the right set of active list links for this direction */ - if (direction == CELL_DIRECTION_OUT) { - next_active = &(circ->next_active_on_n_chan); - prev_active = &(circ->prev_active_on_n_chan); - circuit_cmux = circ->n_mux; - chan = circ->n_chan; - circ_id = circ->n_circ_id; - } else { - next_active = &(TO_OR_CIRCUIT(circ)->next_active_on_p_chan); - prev_active = &(TO_OR_CIRCUIT(circ)->prev_active_on_p_chan); - circuit_cmux = TO_OR_CIRCUIT(circ)->p_mux; - chan = TO_OR_CIRCUIT(circ)->p_chan; - circ_id = TO_OR_CIRCUIT(circ)->p_circ_id; - } - - /* Assert that it is attached to this mux and a channel */ - tor_assert(cmux == circuit_cmux); - tor_assert(chan != NULL); - - /* - * Check if the circuit really was active; if it's inactive, the - * next_active and prev_active pointers will be NULL and this circuit - * will not be the head or tail of the list for this cmux. - */ - already_inactive = (*prev_active == NULL && *next_active == NULL && - cmux->active_circuits_head != circ && - cmux->active_circuits_tail != circ); - - /* If we're already inactive, log a warning and finish */ - if (already_inactive) { - log_warn(LD_CIRC, - "Circuit %d on channel " U64_FORMAT " was already inactive", - (unsigned)circ_id, U64_PRINTF_ARG(chan->global_identifier)); - return; - } - - /* Remove from the list; first get next_prev and prev_next */ - if (*next_active) { - /* - * If there's a next circuit, its previous circuit becomes this - * circuit's previous circuit. - */ - next_prev = circuitmux_prev_active_circ_p(cmux, *next_active); - } else { - /* Else, the tail becomes this circuit's previous circuit */ - next_prev = &(cmux->active_circuits_tail); - } - - /* Got next_prev, now prev_next */ - if (*prev_active) { - /* - * If there's a previous circuit, its next circuit becomes this circuit's - * next circuit. - */ - prev_next = circuitmux_next_active_circ_p(cmux, *prev_active); - } else { - /* Else, the head becomes this circuit's next circuit */ - prev_next = &(cmux->active_circuits_head); - } - - /* Assert that we got sensible values for the next/prev pointers */ - tor_assert(next_prev != NULL); - tor_assert(prev_next != NULL); - - /* Update the next/prev pointers - this removes circ from the list */ - *next_prev = *prev_active; - *prev_next = *next_active; - - /* Now null out prev_active/next_active */ - *prev_active = NULL; - *next_active = NULL; /* Policy-specific notification */ - if (cmux->policy && - cmux->policy->notify_circ_inactive) { + if (cmux->policy->notify_circ_inactive) { /* Okay, we need to check the circuit for policy data now */ - hashent = circuitmux_find_map_entry(cmux, circ); + chanid_circid_muxinfo_t *hashent = circuitmux_find_map_entry(cmux, circ); /* We should have found something */ tor_assert(hashent); /* Notify */ cmux->policy->notify_circ_inactive(cmux, cmux->policy_data, circ, hashent->muxinfo.policy_data); } - - circuitmux_assert_okay_paranoid(cmux); } /** @@ -1400,8 +1073,6 @@ circuitmux_set_num_cells(circuitmux_t *cmux, circuit_t *circ, tor_assert(cmux); tor_assert(circ); - circuitmux_assert_okay_paranoid(cmux); - /* Search for this circuit's entry */ hashent = circuitmux_find_map_entry(cmux, circ); /* Assert that we found one */ @@ -1412,7 +1083,7 @@ circuitmux_set_num_cells(circuitmux_t *cmux, circuit_t *circ, cmux->n_cells += n_cells; /* Do we need to notify a cmux policy? */ - if (cmux->policy && cmux->policy->notify_set_n_cells) { + if (cmux->policy->notify_set_n_cells) { /* Call notify_set_n_cells */ cmux->policy->notify_set_n_cells(cmux, cmux->policy_data, @@ -1428,21 +1099,15 @@ circuitmux_set_num_cells(circuitmux_t *cmux, circuit_t *circ, if (hashent->muxinfo.cell_count > 0 && n_cells == 0) { --(cmux->n_active_circuits); hashent->muxinfo.cell_count = n_cells; - circuitmux_make_circuit_inactive(cmux, circ, hashent->muxinfo.direction); + circuitmux_make_circuit_inactive(cmux, circ); /* Is the old cell count == 0 and the new cell count > 0 ? */ } else if (hashent->muxinfo.cell_count == 0 && n_cells > 0) { ++(cmux->n_active_circuits); hashent->muxinfo.cell_count = n_cells; - circuitmux_make_circuit_active(cmux, circ, hashent->muxinfo.direction); + circuitmux_make_circuit_active(cmux, circ); } else { - /* - * Update the entry cell count like this so we can put a - * circuitmux_assert_okay_paranoid inside make_circuit_(in)active() too. - */ hashent->muxinfo.cell_count = n_cells; } - - circuitmux_assert_okay_paranoid(cmux); } /* @@ -1468,6 +1133,9 @@ circuitmux_get_first_active_circuit(circuitmux_t *cmux, circuit_t *circ = NULL; tor_assert(cmux); + tor_assert(cmux->policy); + /* This callback is mandatory. */ + tor_assert(cmux->policy->pick_active_circuit); tor_assert(destroy_queue_out); *destroy_queue_out = NULL; @@ -1486,14 +1154,7 @@ circuitmux_get_first_active_circuit(circuitmux_t *cmux, /* We also must have a cell available for this to be the case */ tor_assert(cmux->n_cells > 0); /* Do we have a policy-provided circuit selector? */ - if (cmux->policy && cmux->policy->pick_active_circuit) { - circ = cmux->policy->pick_active_circuit(cmux, cmux->policy_data); - } - /* Fall back on the head of the active circuits list */ - if (!circ) { - tor_assert(cmux->active_circuits_head); - circ = cmux->active_circuits_head; - } + circ = cmux->policy->pick_active_circuit(cmux, cmux->policy_data); cmux->last_cell_was_destroy = 0; } else { tor_assert(cmux->n_cells == 0); @@ -1517,7 +1178,6 @@ circuitmux_notify_xmit_cells(circuitmux_t *cmux, circuit_t *circ, tor_assert(cmux); tor_assert(circ); - circuitmux_assert_okay_paranoid(cmux); if (n_cells == 0) return; @@ -1544,17 +1204,11 @@ circuitmux_notify_xmit_cells(circuitmux_t *cmux, circuit_t *circ, /* Adjust the mux cell counter */ cmux->n_cells -= n_cells; - /* If we aren't making it inactive later, move it to the tail of the list */ - if (!becomes_inactive) { - circuitmux_move_active_circ_to_tail(cmux, circ, - hashent->muxinfo.direction); - } - /* * We call notify_xmit_cells() before making the circuit inactive if needed, * so the policy can always count on this coming in on an active circuit. */ - if (cmux->policy && cmux->policy->notify_xmit_cells) { + if (cmux->policy->notify_xmit_cells) { cmux->policy->notify_xmit_cells(cmux, cmux->policy_data, circ, hashent->muxinfo.policy_data, n_cells); @@ -1566,10 +1220,8 @@ circuitmux_notify_xmit_cells(circuitmux_t *cmux, circuit_t *circ, */ if (becomes_inactive) { --(cmux->n_active_circuits); - circuitmux_make_circuit_inactive(cmux, circ, hashent->muxinfo.direction); + circuitmux_make_circuit_inactive(cmux, circ); } - - circuitmux_assert_okay_paranoid(cmux); } /** @@ -1592,282 +1244,6 @@ circuitmux_notify_xmit_destroy(circuitmux_t *cmux) I64_PRINTF_ARG(global_destroy_ctr)); } -/* - * Circuitmux consistency checking assertions - */ - -/** - * Check that circuitmux data structures are consistent and fail with an - * assert if not. - */ - -void -circuitmux_assert_okay(circuitmux_t *cmux) -{ - tor_assert(cmux); - - /* - * Pass 1: iterate the hash table; for each entry: - * a) Check that the circuit has this cmux for n_mux or p_mux - * b) If the cell_count is > 0, set the mark bit; otherwise clear it - * c) Also check activeness (cell_count > 0 should be active) - * d) Count the number of circuits, active circuits and queued cells - * and at the end check that they match the counters in the cmux. - * - * Pass 2: iterate the active circuits list; for each entry, - * make sure the circuit is attached to this mux and appears - * in the hash table. Make sure the mark bit is 1, and clear - * it in the hash table entry. Consistency-check the linked - * list pointers. - * - * Pass 3: iterate the hash table again; assert if any active circuits - * (mark bit set to 1) are discovered that weren't cleared in pass 2 - * (don't appear in the linked list). - */ - - circuitmux_assert_okay_pass_one(cmux); - circuitmux_assert_okay_pass_two(cmux); - circuitmux_assert_okay_pass_three(cmux); -} - -/** - * Do the first pass of circuitmux_assert_okay(); see the comment in that - * function. - */ - -static void -circuitmux_assert_okay_pass_one(circuitmux_t *cmux) -{ - chanid_circid_muxinfo_t **i = NULL; - uint64_t chan_id; - channel_t *chan; - circid_t circ_id; - circuit_t *circ; - or_circuit_t *or_circ; - circuit_t **next_p, **prev_p; - unsigned int n_circuits, n_active_circuits, n_cells; - - tor_assert(cmux); - tor_assert(cmux->chanid_circid_map); - - /* Reset the counters */ - n_circuits = n_active_circuits = n_cells = 0; - /* Start iterating the hash table */ - i = HT_START(chanid_circid_muxinfo_map, cmux->chanid_circid_map); - while (i) { - /* Assert that the hash table entry isn't null */ - tor_assert(*i); - - /* Get the channel and circuit id */ - chan_id = (*i)->chan_id; - circ_id = (*i)->circ_id; - - /* Find the channel and circuit, assert that they exist */ - chan = channel_find_by_global_id(chan_id); - tor_assert(chan); - circ = circuit_get_by_circid_channel_even_if_marked(circ_id, chan); - tor_assert(circ); - - /* Assert that we know which direction this is going */ - tor_assert((*i)->muxinfo.direction == CELL_DIRECTION_OUT || - (*i)->muxinfo.direction == CELL_DIRECTION_IN); - - if ((*i)->muxinfo.direction == CELL_DIRECTION_OUT) { - /* We should be n_mux on this circuit */ - tor_assert(cmux == circ->n_mux); - tor_assert(chan == circ->n_chan); - /* Get next and prev for next test */ - next_p = &(circ->next_active_on_n_chan); - prev_p = &(circ->prev_active_on_n_chan); - } else { - /* This should be an or_circuit_t and we should be p_mux */ - or_circ = TO_OR_CIRCUIT(circ); - tor_assert(cmux == or_circ->p_mux); - tor_assert(chan == or_circ->p_chan); - /* Get next and prev for next test */ - next_p = &(or_circ->next_active_on_p_chan); - prev_p = &(or_circ->prev_active_on_p_chan); - } - - /* - * Should this circuit be active? I.e., does the mux know about > 0 - * cells on it? - */ - const int circ_is_active = ((*i)->muxinfo.cell_count > 0); - - /* It should be in the linked list iff it's active */ - if (circ_is_active) { - /* Either we have a next link or we are the tail */ - tor_assert(*next_p || (circ == cmux->active_circuits_tail)); - /* Either we have a prev link or we are the head */ - tor_assert(*prev_p || (circ == cmux->active_circuits_head)); - /* Increment the active circuits counter */ - ++n_active_circuits; - } else { - /* Shouldn't be in list, so no next or prev link */ - tor_assert(!(*next_p)); - tor_assert(!(*prev_p)); - /* And can't be head or tail */ - tor_assert(circ != cmux->active_circuits_head); - tor_assert(circ != cmux->active_circuits_tail); - } - - /* Increment the circuits counter */ - ++n_circuits; - /* Adjust the cell counter */ - n_cells += (*i)->muxinfo.cell_count; - - /* Set the mark bit to circ_is_active */ - (*i)->muxinfo.mark = circ_is_active; - - /* Advance to the next entry */ - i = HT_NEXT(chanid_circid_muxinfo_map, cmux->chanid_circid_map, i); - } - - /* Now check the counters */ - tor_assert(n_cells == cmux->n_cells); - tor_assert(n_circuits == cmux->n_circuits); - tor_assert(n_active_circuits == cmux->n_active_circuits); -} - -/** - * Do the second pass of circuitmux_assert_okay(); see the comment in that - * function. - */ - -static void -circuitmux_assert_okay_pass_two(circuitmux_t *cmux) -{ - circuit_t *curr_circ, *prev_circ = NULL, *next_circ; - or_circuit_t *curr_or_circ; - uint64_t curr_chan_id; - circid_t curr_circ_id; - circuit_t **next_p, **prev_p; - channel_t *chan; - unsigned int n_active_circuits = 0; - chanid_circid_muxinfo_t search, *hashent = NULL; - - tor_assert(cmux); - tor_assert(cmux->chanid_circid_map); - - /* - * Walk the linked list of active circuits in cmux; keep track of the - * previous circuit seen for consistency checking purposes. Count them - * to make sure the number in the linked list matches - * cmux->n_active_circuits. - */ - curr_circ = cmux->active_circuits_head; - while (curr_circ) { - /* Reset some things */ - chan = NULL; - curr_or_circ = NULL; - next_circ = NULL; - next_p = prev_p = NULL; - cell_direction_t direction; - - /* Figure out if this is n_mux or p_mux */ - if (cmux == curr_circ->n_mux) { - /* Get next_p and prev_p */ - next_p = &(curr_circ->next_active_on_n_chan); - prev_p = &(curr_circ->prev_active_on_n_chan); - /* Get the channel */ - chan = curr_circ->n_chan; - /* Get the circuit id */ - curr_circ_id = curr_circ->n_circ_id; - /* Remember the direction */ - direction = CELL_DIRECTION_OUT; - } else { - /* We must be p_mux and this must be an or_circuit_t */ - curr_or_circ = TO_OR_CIRCUIT(curr_circ); - tor_assert(cmux == curr_or_circ->p_mux); - /* Get next_p and prev_p */ - next_p = &(curr_or_circ->next_active_on_p_chan); - prev_p = &(curr_or_circ->prev_active_on_p_chan); - /* Get the channel */ - chan = curr_or_circ->p_chan; - /* Get the circuit id */ - curr_circ_id = curr_or_circ->p_circ_id; - /* Remember the direction */ - direction = CELL_DIRECTION_IN; - } - - /* Assert that we got a channel and get the channel ID */ - tor_assert(chan); - curr_chan_id = chan->global_identifier; - - /* Assert that prev_p points to last circuit we saw */ - tor_assert(*prev_p == prev_circ); - /* If that's NULL, assert that we are the head */ - if (!(*prev_p)) tor_assert(curr_circ == cmux->active_circuits_head); - - /* Get the next circuit */ - next_circ = *next_p; - /* If it's NULL, assert that we are the tail */ - if (!(*next_p)) tor_assert(curr_circ == cmux->active_circuits_tail); - - /* Now find the hash table entry for this circuit */ - search.chan_id = curr_chan_id; - search.circ_id = curr_circ_id; - hashent = HT_FIND(chanid_circid_muxinfo_map, cmux->chanid_circid_map, - &search); - - /* Assert that we have one */ - tor_assert(hashent); - - /* Assert that the direction matches */ - tor_assert(direction == hashent->muxinfo.direction); - - /* Assert that the hash entry got marked in pass one */ - tor_assert(hashent->muxinfo.mark); - - /* Clear the mark */ - hashent->muxinfo.mark = 0; - - /* Increment the counter */ - ++n_active_circuits; - - /* Advance to the next active circuit and update prev_circ */ - prev_circ = curr_circ; - curr_circ = next_circ; - } - - /* Assert that the counter matches the cmux */ - tor_assert(n_active_circuits == cmux->n_active_circuits); -} - -/** - * Do the third pass of circuitmux_assert_okay(); see the comment in that - * function. - */ - -static void -circuitmux_assert_okay_pass_three(circuitmux_t *cmux) -{ - chanid_circid_muxinfo_t **i = NULL; - - tor_assert(cmux); - tor_assert(cmux->chanid_circid_map); - - /* Start iterating the hash table */ - i = HT_START(chanid_circid_muxinfo_map, cmux->chanid_circid_map); - - /* Advance through each entry */ - while (i) { - /* Assert that it isn't null */ - tor_assert(*i); - - /* - * Assert that this entry is not marked - i.e., that either we didn't - * think it should be active in pass one or we saw it in the active - * circuits linked list. - */ - tor_assert(!((*i)->muxinfo.mark)); - - /* Advance to the next entry */ - i = HT_NEXT(chanid_circid_muxinfo_map, cmux->chanid_circid_map, i); - } -} - /*DOCDOC */ void circuitmux_append_destroy_cell(channel_t *chan, diff --git a/src/or/circuitmux_ewma.c b/src/or/circuitmux_ewma.c index fde2d22a89..e5d5a14581 100644 --- a/src/or/circuitmux_ewma.c +++ b/src/or/circuitmux_ewma.c @@ -28,7 +28,7 @@ * **/ -#define TOR_CIRCUITMUX_EWMA_C_ +#define CIRCUITMUX_EWMA_PRIVATE #include "orconfig.h" @@ -37,6 +37,7 @@ #include "or.h" #include "circuitmux.h" #include "circuitmux_ewma.h" +#include "crypto_rand.h" #include "networkstatus.h" /*** EWMA parameter #defines ***/ @@ -169,8 +170,6 @@ TO_EWMA_POL_CIRC_DATA(circuitmux_policy_circ_data_t *pol) static void add_cell_ewma(ewma_policy_data_t *pol, cell_ewma_t *ewma); static int compare_cell_ewma_counts(const void *p1, const void *p2); -static unsigned cell_ewma_tick_from_timeval(const struct timeval *now, - double *remainder_out); static circuit_t * cell_ewma_to_circuit(cell_ewma_t *ewma); static inline double get_scale_factor(unsigned from_tick, unsigned to_tick); static cell_ewma_t * pop_first_cell_ewma(ewma_policy_data_t *pol); @@ -223,8 +222,6 @@ ewma_cmp_cmux(circuitmux_t *cmux_1, circuitmux_policy_data_t *pol_data_1, * has value ewma_scale_factor ** N.) */ static double ewma_scale_factor = 0.1; -/* DOCDOC ewma_enabled */ -static int ewma_enabled = 0; /*** EWMA circuitmux_policy_t method table ***/ @@ -241,8 +238,26 @@ circuitmux_policy_t ewma_policy = { /*.cmp_cmux =*/ ewma_cmp_cmux }; +/** Have we initialized the ewma tick-counting logic? */ +static int ewma_ticks_initialized = 0; +/** At what monotime_coarse_t did the current tick begin? */ +static monotime_coarse_t start_of_current_tick; +/** What is the number of the current tick? */ +static unsigned current_tick_num; + /*** EWMA method implementations using the below EWMA helper functions ***/ +/** Compute and return the current cell_ewma tick. */ +static inline unsigned int +cell_ewma_get_tick(void) +{ + monotime_coarse_t now; + monotime_coarse_get(&now); + int32_t msec_diff = monotime_coarse_diff_msec32(&start_of_current_tick, + &now); + return current_tick_num + msec_diff / (1000*EWMA_TICK_LEN); +} + /** * Allocate an ewma_policy_data_t and upcast it to a circuitmux_policy_data_t; * this is called when setting the policy on a circuitmux_t to ewma_policy. @@ -416,8 +431,6 @@ ewma_notify_xmit_cells(circuitmux_t *cmux, ewma_policy_circ_data_t *cdata = NULL; unsigned int tick; double fractional_tick, ewma_increment; - /* The current (hi-res) time */ - struct timeval now_hires; cell_ewma_t *cell_ewma, *tmp; tor_assert(cmux); @@ -430,8 +443,7 @@ ewma_notify_xmit_cells(circuitmux_t *cmux, cdata = TO_EWMA_POL_CIRC_DATA(pol_circ_data); /* Rescale the EWMAs if needed */ - tor_gettimeofday_cached(&now_hires); - tick = cell_ewma_tick_from_timeval(&now_hires, &fractional_tick); + tick = cell_ewma_get_current_tick_and_fraction(&fractional_tick); if (tick != pol->active_circuit_pqueue_last_recalibrated) { scale_active_circuits(pol, tick); @@ -592,79 +604,122 @@ cell_ewma_to_circuit(cell_ewma_t *ewma) rescale. */ -/** Given a timeval <b>now</b>, compute the cell_ewma tick in which it occurs - * and the fraction of the tick that has elapsed between the start of the tick - * and <b>now</b>. Return the former and store the latter in - * *<b>remainder_out</b>. +/** + * Initialize the system that tells which ewma tick we are in. + */ +STATIC void +cell_ewma_initialize_ticks(void) +{ + if (ewma_ticks_initialized) + return; + monotime_coarse_get(&start_of_current_tick); + crypto_rand((char*)¤t_tick_num, sizeof(current_tick_num)); + ewma_ticks_initialized = 1; +} + +/** Compute the current cell_ewma tick and the fraction of the tick that has + * elapsed between the start of the tick and the current time. Return the + * former and store the latter in *<b>remainder_out</b>. * * These tick values are not meant to be shared between Tor instances, or used * for other purposes. */ - -static unsigned -cell_ewma_tick_from_timeval(const struct timeval *now, - double *remainder_out) +STATIC unsigned +cell_ewma_get_current_tick_and_fraction(double *remainder_out) { - unsigned res = (unsigned) (now->tv_sec / EWMA_TICK_LEN); - /* rem */ - double rem = (now->tv_sec % EWMA_TICK_LEN) + - ((double)(now->tv_usec)) / 1.0e6; - *remainder_out = rem / EWMA_TICK_LEN; - return res; + if (BUG(!ewma_ticks_initialized)) { + cell_ewma_initialize_ticks(); // LCOV_EXCL_LINE + } + monotime_coarse_t now; + monotime_coarse_get(&now); + int32_t msec_diff = monotime_coarse_diff_msec32(&start_of_current_tick, + &now); + if (msec_diff > (1000*EWMA_TICK_LEN)) { + unsigned ticks_difference = msec_diff / (1000*EWMA_TICK_LEN); + monotime_coarse_add_msec(&start_of_current_tick, + &start_of_current_tick, + ticks_difference * 1000 * EWMA_TICK_LEN); + current_tick_num += ticks_difference; + msec_diff %= 1000*EWMA_TICK_LEN; + } + *remainder_out = ((double)msec_diff) / (1.0e3 * EWMA_TICK_LEN); + return current_tick_num; } -/** Tell the caller whether ewma_enabled is set */ -int -cell_ewma_enabled(void) +/* Default value for the CircuitPriorityHalflifeMsec consensus parameter in + * msec. */ +#define CMUX_PRIORITY_HALFLIFE_MSEC_DEFAULT 30000 +/* Minimum and maximum value for the CircuitPriorityHalflifeMsec consensus + * parameter. */ +#define CMUX_PRIORITY_HALFLIFE_MSEC_MIN 1 +#define CMUX_PRIORITY_HALFLIFE_MSEC_MAX INT32_MAX + +/* Return the value of the circuit priority halflife from the options if + * available or else from the consensus (in that order). If none can be found, + * a default value is returned. + * + * The source_msg points to a string describing from where the value was + * picked so it can be used for logging. */ +static double +get_circuit_priority_halflife(const or_options_t *options, + const networkstatus_t *consensus, + const char **source_msg) { - return ewma_enabled; -} + int32_t halflife_ms; + double halflife; + /* Compute the default value now. We might need it. */ + double halflife_default = + ((double) CMUX_PRIORITY_HALFLIFE_MSEC_DEFAULT) / 1000.0; -/** Compute and return the current cell_ewma tick. */ -unsigned int -cell_ewma_get_tick(void) -{ - return ((unsigned)approx_time() / EWMA_TICK_LEN); + /* Try to get it from configuration file first. */ + if (options && options->CircuitPriorityHalflife >= -EPSILON) { + halflife = options->CircuitPriorityHalflife; + *source_msg = "CircuitPriorityHalflife in configuration"; + goto end; + } + + /* Try to get the msec value from the consensus. */ + halflife_ms = networkstatus_get_param(consensus, + "CircuitPriorityHalflifeMsec", + CMUX_PRIORITY_HALFLIFE_MSEC_DEFAULT, + CMUX_PRIORITY_HALFLIFE_MSEC_MIN, + CMUX_PRIORITY_HALFLIFE_MSEC_MAX); + halflife = ((double) halflife_ms) / 1000.0; + *source_msg = "CircuitPriorityHalflifeMsec in consensus"; + + end: + /* We should never go below the EPSILON else we would consider it disabled + * and we can't have that. */ + if (halflife < EPSILON) { + log_warn(LD_CONFIG, "CircuitPriorityHalflife is too small (%f). " + "Adjusting to the smallest value allowed: %f.", + halflife, halflife_default); + halflife = halflife_default; + } + return halflife; } /** Adjust the global cell scale factor based on <b>options</b> */ void -cell_ewma_set_scale_factor(const or_options_t *options, - const networkstatus_t *consensus) +cmux_ewma_set_options(const or_options_t *options, + const networkstatus_t *consensus) { - int32_t halflife_ms; double halflife; const char *source; - if (options && options->CircuitPriorityHalflife >= -EPSILON) { - halflife = options->CircuitPriorityHalflife; - source = "CircuitPriorityHalflife in configuration"; - } else if (consensus && (halflife_ms = networkstatus_get_param( - consensus, "CircuitPriorityHalflifeMsec", - -1, -1, INT32_MAX)) >= 0) { - halflife = ((double)halflife_ms)/1000.0; - source = "CircuitPriorityHalflifeMsec in consensus"; - } else { - halflife = EWMA_DEFAULT_HALFLIFE; - source = "Default value"; - } - if (halflife <= EPSILON) { - /* The cell EWMA algorithm is disabled. */ - ewma_scale_factor = 0.1; - ewma_enabled = 0; - log_info(LD_OR, - "Disabled cell_ewma algorithm because of value in %s", - source); - } else { - /* convert halflife into halflife-per-tick. */ - halflife /= EWMA_TICK_LEN; - /* compute per-tick scale factor. */ - ewma_scale_factor = exp( LOG_ONEHALF / halflife ); - ewma_enabled = 1; - log_info(LD_OR, - "Enabled cell_ewma algorithm because of value in %s; " - "scale factor is %f per %d seconds", - source, ewma_scale_factor, EWMA_TICK_LEN); - } + cell_ewma_initialize_ticks(); + + /* Both options and consensus can be NULL. This assures us to either get a + * valid configured value or the default one. */ + halflife = get_circuit_priority_halflife(options, consensus, &source); + + /* convert halflife into halflife-per-tick. */ + halflife /= EWMA_TICK_LEN; + /* compute per-tick scale factor. */ + ewma_scale_factor = exp( LOG_ONEHALF / halflife ); + log_info(LD_OR, + "Enabled cell_ewma algorithm because of value in %s; " + "scale factor is %f per %d seconds", + source, ewma_scale_factor, EWMA_TICK_LEN); } /** Return the multiplier necessary to convert the value of a cell sent in @@ -763,3 +818,12 @@ pop_first_cell_ewma(ewma_policy_data_t *pol) offsetof(cell_ewma_t, heap_index)); } +/** + * Drop all resources held by circuitmux_ewma.c, and deinitialize the + * module. */ +void +circuitmux_ewma_free_all(void) +{ + ewma_ticks_initialized = 0; +} + diff --git a/src/or/circuitmux_ewma.h b/src/or/circuitmux_ewma.h index 8f4e57865e..f0c4c36095 100644 --- a/src/or/circuitmux_ewma.h +++ b/src/or/circuitmux_ewma.h @@ -12,13 +12,19 @@ #include "or.h" #include "circuitmux.h" +/* The public EWMA policy callbacks object. */ extern circuitmux_policy_t ewma_policy; /* Externally visible EWMA functions */ -int cell_ewma_enabled(void); -unsigned int cell_ewma_get_tick(void); -void cell_ewma_set_scale_factor(const or_options_t *options, - const networkstatus_t *consensus); +void cmux_ewma_set_options(const or_options_t *options, + const networkstatus_t *consensus); + +void circuitmux_ewma_free_all(void); + +#ifdef CIRCUITMUX_EWMA_PRIVATE +STATIC unsigned cell_ewma_get_current_tick_and_fraction(double *remainder_out); +STATIC void cell_ewma_initialize_ticks(void); +#endif #endif /* !defined(TOR_CIRCUITMUX_EWMA_H) */ diff --git a/src/or/circuitstats.c b/src/or/circuitstats.c index f06c2e5e38..94f75c590f 100644 --- a/src/or/circuitstats.c +++ b/src/or/circuitstats.c @@ -31,10 +31,12 @@ #include "config.h" #include "confparse.h" #include "control.h" +#include "crypto_rand.h" #include "main.h" #include "networkstatus.h" #include "rendclient.h" #include "rendservice.h" +#include "router.h" #include "statefile.h" #include "circuitlist.h" #include "circuituse.h" @@ -125,7 +127,7 @@ circuit_build_times_disabled_(const or_options_t *options, ignore_consensus ? 0 : networkstatus_get_param(NULL, "cbtdisabled", 0, 0, 1); int config_disabled = !options->LearnCircuitBuildTimeout; - int dirauth_disabled = options->AuthoritativeDir; + int dirauth_disabled = authdir_mode(options); int state_disabled = did_last_state_file_write_fail() ? 1 : 0; /* LearnCircuitBuildTimeout and Tor2web/Single Onion Services are * incompatible in two ways: diff --git a/src/or/circuituse.c b/src/or/circuituse.c index dc62e4d09c..8e007ce920 100644 --- a/src/or/circuituse.c +++ b/src/or/circuituse.c @@ -55,7 +55,6 @@ #include "rephist.h" #include "router.h" #include "routerlist.h" -#include "config.h" static void circuit_expire_old_circuits_clientside(void); static void circuit_increment_failure_count(void); @@ -1632,7 +1631,7 @@ circuit_testing_opened(origin_circuit_t *circ) router_perform_bandwidth_test(NUM_PARALLEL_TESTING_CIRCS, time(NULL)); have_performed_bandwidth_test = 1; } else - consider_testing_reachability(1, 0); + router_do_reachability_checks(1, 0); } /** A testing circuit has failed to build. Take whatever stats we want. */ @@ -2607,7 +2606,7 @@ link_apconn_to_circ(entry_connection_t *apconn, origin_circuit_t *circ, log_debug(LD_APP|LD_CIRC, "attaching new conn to circ. n_circ_id %u.", (unsigned)circ->base_.n_circ_id); /* reset it, so we can measure circ timeouts */ - ENTRY_TO_CONN(apconn)->timestamp_lastread = time(NULL); + ENTRY_TO_CONN(apconn)->timestamp_last_read_allowed = time(NULL); ENTRY_TO_EDGE_CONN(apconn)->next_stream = circ->p_streams; ENTRY_TO_EDGE_CONN(apconn)->on_circuit = TO_CIRCUIT(circ); /* assert_connection_ok(conn, time(NULL)); */ @@ -3107,3 +3106,41 @@ mark_circuit_unusable_for_new_conns(origin_circuit_t *circ) circ->unusable_for_new_conns = 1; } +/** + * Add relay_body_len and RELAY_PAYLOAD_SIZE-relay_body_len to + * the valid delivered written fields and the overhead field, + * respectively. + */ +void +circuit_sent_valid_data(origin_circuit_t *circ, uint16_t relay_body_len) +{ + if (!circ) return; + + tor_assert_nonfatal(relay_body_len <= RELAY_PAYLOAD_SIZE); + + circ->n_delivered_written_circ_bw = + tor_add_u32_nowrap(circ->n_delivered_written_circ_bw, relay_body_len); + circ->n_overhead_written_circ_bw = + tor_add_u32_nowrap(circ->n_overhead_written_circ_bw, + RELAY_PAYLOAD_SIZE-relay_body_len); +} + +/** + * Add relay_body_len and RELAY_PAYLOAD_SIZE-relay_body_len to + * the valid delivered read field and the overhead field, + * respectively. + */ +void +circuit_read_valid_data(origin_circuit_t *circ, uint16_t relay_body_len) +{ + if (!circ) return; + + tor_assert_nonfatal(relay_body_len <= RELAY_PAYLOAD_SIZE); + + circ->n_delivered_read_circ_bw = + tor_add_u32_nowrap(circ->n_delivered_read_circ_bw, relay_body_len); + circ->n_overhead_read_circ_bw = + tor_add_u32_nowrap(circ->n_overhead_read_circ_bw, + RELAY_PAYLOAD_SIZE-relay_body_len); +} + diff --git a/src/or/circuituse.h b/src/or/circuituse.h index 71c818b978..6458bd6908 100644 --- a/src/or/circuituse.h +++ b/src/or/circuituse.h @@ -65,6 +65,8 @@ void mark_circuit_unusable_for_new_conns(origin_circuit_t *circ); int circuit_purpose_is_hidden_service(uint8_t); int circuit_should_use_vanguards(uint8_t); +void circuit_sent_valid_data(origin_circuit_t *circ, uint16_t relay_body_len); +void circuit_read_valid_data(origin_circuit_t *circ, uint16_t relay_body_len); #ifdef TOR_UNIT_TESTS /* Used only by circuituse.c and test_circuituse.c */ diff --git a/src/or/command.c b/src/or/command.c index 7280be1396..39950f41bf 100644 --- a/src/or/command.c +++ b/src/or/command.c @@ -46,6 +46,7 @@ #include "config.h" #include "control.h" #include "cpuworker.h" +#include "crypto_util.h" #include "dos.h" #include "hibernate.h" #include "nodelist.h" @@ -339,7 +340,9 @@ command_process_create_cell(cell_t *cell, channel_t *chan) return; } - if (connection_or_digest_is_known_relay(chan->identity_digest)) { + if (!channel_is_client(chan)) { + /* remember create types we've seen, but don't remember them from + * clients, to be extra conservative about client statistics. */ rep_hist_note_circuit_handshake_requested(create_cell->handshake_type); } @@ -493,6 +496,17 @@ command_process_relay_cell(cell_t *cell, channel_t *chan) /* if we're a relay and treating connections with recent local * traffic better, then this is one of them. */ channel_timestamp_client(chan); + + /* Count all circuit bytes here for control port accuracy. We want + * to count even invalid/dropped relay cells, hence counting + * before the recognized check and the connection_edge_process_relay + * cell checks. + */ + origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); + + /* Count the payload bytes only. We don't care about cell headers */ + ocirc->n_read_circ_bw = tor_add_u32_nowrap(ocirc->n_read_circ_bw, + CELL_PAYLOAD_SIZE); } if (!CIRCUIT_IS_ORIGIN(circ) && diff --git a/src/or/config.c b/src/or/config.c index 58080c65e3..2660fbd787 100644 --- a/src/or/config.c +++ b/src/or/config.c @@ -1,3 +1,4 @@ + /* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. @@ -78,8 +79,9 @@ #include "control.h" #include "confparse.h" #include "cpuworker.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "dirserv.h" -#include "dirvote.h" #include "dns.h" #include "dos.h" #include "entrynodes.h" @@ -104,12 +106,16 @@ #include "statefile.h" #include "transports.h" #include "ext_orport.h" +#include "voting_schedule.h" #ifdef _WIN32 #include <shlobj.h> #endif #include "procmon.h" +#include "dirauth/dirvote.h" +#include "dirauth/mode.h" + #ifdef HAVE_SYSTEMD # if defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__) /* Systemd's use of gcc's __INCLUDE_LEVEL__ extension macro appears to confuse @@ -126,6 +132,11 @@ static const char unix_socket_prefix[] = "unix:"; * configuration. */ static const char unix_q_socket_prefix[] = "unix:\""; +/** macro to help with the bulk rename of *DownloadSchedule to + * *DowloadInitialDelay . */ +#define DOWNLOAD_SCHEDULE(name) \ + { #name "DownloadSchedule", #name "DownloadInitialDelay", 0, 1 } + /** A list of abbreviations and aliases to map command-line options, obsolete * option names, or alternative option names, to their current values. */ static config_abbrev_t option_abbrevs_[] = { @@ -175,6 +186,16 @@ static config_abbrev_t option_abbrevs_[] = { { "_HSLayer2Nodes", "HSLayer2Nodes", 0, 1 }, { "_HSLayer3Nodes", "HSLayer3Nodes", 0, 1 }, + DOWNLOAD_SCHEDULE(ClientBootstrapConsensusAuthority), + DOWNLOAD_SCHEDULE(ClientBootstrapConsensusAuthorityOnly), + DOWNLOAD_SCHEDULE(ClientBootstrapConsensusFallback), + DOWNLOAD_SCHEDULE(TestingBridge), + DOWNLOAD_SCHEDULE(TestingBridgeBootstrap), + DOWNLOAD_SCHEDULE(TestingClient), + DOWNLOAD_SCHEDULE(TestingClientConsensus), + DOWNLOAD_SCHEDULE(TestingServer), + DOWNLOAD_SCHEDULE(TestingServerConsensus), + { NULL, NULL, 0, 0}, }; @@ -267,7 +288,7 @@ static config_var_t option_vars_[] = { OBSOLETE("CircuitIdleTimeout"), V(CircuitsAvailableTimeout, INTERVAL, "0"), V(CircuitStreamTimeout, INTERVAL, "0"), - V(CircuitPriorityHalflife, DOUBLE, "-100.0"), /*negative:'Use default'*/ + V(CircuitPriorityHalflife, DOUBLE, "-1.0"), /*negative:'Use default'*/ V(ClientDNSRejectInternalAddresses, BOOL,"1"), V(ClientOnly, BOOL, "0"), V(ClientPreferIPv6ORPort, AUTOBOOL, "auto"), @@ -337,7 +358,7 @@ static config_var_t option_vars_[] = { V(DownloadExtraInfo, BOOL, "0"), V(TestingEnableConnBwEvent, BOOL, "0"), V(TestingEnableCellStatsEvent, BOOL, "0"), - V(TestingEnableTbEmptyEvent, BOOL, "0"), + OBSOLETE("TestingEnableTbEmptyEvent"), V(EnforceDistinctSubnets, BOOL, "1"), V(EntryNodes, ROUTERSET, NULL), V(EntryStatistics, BOOL, "0"), @@ -457,6 +478,7 @@ static config_var_t option_vars_[] = { V(NumCPUs, UINT, "0"), V(NumDirectoryGuards, UINT, "0"), V(NumEntryGuards, UINT, "0"), + V(NumPrimaryGuards, UINT, "0"), V(OfflineMasterKey, BOOL, "0"), OBSOLETE("ORListenAddress"), VPORT(ORPort), @@ -495,8 +517,8 @@ static config_var_t option_vars_[] = { V(TestingSigningKeySlop, INTERVAL, "1 day"), V(OptimisticData, AUTOBOOL, "auto"), - V(PortForwarding, BOOL, "0"), - V(PortForwardingHelper, FILENAME, "tor-fw-helper"), + OBSOLETE("PortForwarding"), + OBSOLETE("PortForwardingHelper"), OBSOLETE("PreferTunneledDirConns"), V(ProtocolWarnings, BOOL, "0"), V(PublishServerDescriptor, CSV, "1"), @@ -599,16 +621,10 @@ static config_var_t option_vars_[] = { VAR("__OwningControllerProcess",STRING,OwningControllerProcess, NULL), VAR("__OwningControllerFD",INT,OwningControllerFD, "-1"), V(MinUptimeHidServDirectoryV2, INTERVAL, "96 hours"), - V(TestingServerDownloadSchedule, CSV_INTERVAL, "0, 0, 0, 60, 60, 120, " - "300, 900, 2147483647"), - V(TestingClientDownloadSchedule, CSV_INTERVAL, "0, 0, 60, 300, 600, " - "2147483647"), - V(TestingServerConsensusDownloadSchedule, CSV_INTERVAL, "0, 0, 60, " - "300, 600, 1800, 1800, 1800, 1800, " - "1800, 3600, 7200"), - V(TestingClientConsensusDownloadSchedule, CSV_INTERVAL, "0, 0, 60, " - "300, 600, 1800, 3600, 3600, 3600, " - "10800, 21600, 43200"), + V(TestingServerDownloadInitialDelay, CSV_INTERVAL, "0"), + V(TestingClientDownloadInitialDelay, CSV_INTERVAL, "0"), + V(TestingServerConsensusDownloadInitialDelay, CSV_INTERVAL, "0"), + V(TestingClientConsensusDownloadInitialDelay, CSV_INTERVAL, "0"), /* With the ClientBootstrapConsensus*Download* below: * Clients with only authorities will try: * - at least 3 authorities over 10 seconds, then exponentially backoff, @@ -624,13 +640,11 @@ static config_var_t option_vars_[] = { * * When clients have authorities and fallbacks available, they use these * schedules: (we stagger the times to avoid thundering herds) */ - V(ClientBootstrapConsensusAuthorityDownloadSchedule, CSV_INTERVAL, - "6, 11, 3600, 10800, 25200, 54000, 111600, 262800" /* 3 days + 1 hour */), - V(ClientBootstrapConsensusFallbackDownloadSchedule, CSV_INTERVAL, - "0, 1, 4, 11, 3600, 10800, 25200, 54000, 111600, 262800"), + V(ClientBootstrapConsensusAuthorityDownloadInitialDelay, CSV_INTERVAL, "6"), + V(ClientBootstrapConsensusFallbackDownloadInitialDelay, CSV_INTERVAL, "0"), /* When clients only have authorities available, they use this schedule: */ - V(ClientBootstrapConsensusAuthorityOnlyDownloadSchedule, CSV_INTERVAL, - "0, 3, 7, 3600, 10800, 25200, 54000, 111600, 262800"), + V(ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay, CSV_INTERVAL, + "0"), /* We don't want to overwhelm slow networks (or mirrors whose replies are * blocked), but we also don't want to fail if only some mirrors are * blackholed. Clients will try 3 directories simultaneously. @@ -638,14 +652,12 @@ static config_var_t option_vars_[] = { V(ClientBootstrapConsensusMaxInProgressTries, UINT, "3"), /* When a client has any running bridges, check each bridge occasionally, * whether or not that bridge is actually up. */ - V(TestingBridgeDownloadSchedule, CSV_INTERVAL, - "10800, 25200, 54000, 111600, 262800"), + V(TestingBridgeDownloadInitialDelay, CSV_INTERVAL,"10800"), /* When a client is just starting, or has no running bridges, check each * bridge a few times quickly, and then try again later. These schedules * are much longer than the other schedules, because we try each and every * configured bridge with this schedule. */ - V(TestingBridgeBootstrapDownloadSchedule, CSV_INTERVAL, - "0, 30, 90, 600, 3600, 10800, 25200, 54000, 111600, 262800"), + V(TestingBridgeBootstrapDownloadInitialDelay, CSV_INTERVAL, "0"), V(TestingClientMaxIntervalWithoutRequest, INTERVAL, "10 minutes"), V(TestingDirConnectionMaxStall, INTERVAL, "5 minutes"), OBSOLETE("TestingConsensusMaxDownloadTries"), @@ -672,12 +684,10 @@ static const config_var_t testing_tor_network_defaults[] = { V(EnforceDistinctSubnets, BOOL, "0"), V(AssumeReachable, BOOL, "1"), V(AuthDirMaxServersPerAddr, UINT, "0"), - V(ClientBootstrapConsensusAuthorityDownloadSchedule, CSV_INTERVAL, - "0, 2, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 8, 16, 32, 60"), - V(ClientBootstrapConsensusFallbackDownloadSchedule, CSV_INTERVAL, - "0, 1, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 8, 16, 32, 60"), - V(ClientBootstrapConsensusAuthorityOnlyDownloadSchedule, CSV_INTERVAL, - "0, 1, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 8, 16, 32, 60"), + V(ClientBootstrapConsensusAuthorityDownloadInitialDelay, CSV_INTERVAL, "0"), + V(ClientBootstrapConsensusFallbackDownloadInitialDelay, CSV_INTERVAL, "0"), + V(ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay, CSV_INTERVAL, + "0"), V(ClientDNSRejectInternalAddresses, BOOL,"0"), V(ClientRejectInternalAddresses, BOOL, "0"), V(CountPrivateBandwidth, BOOL, "1"), @@ -692,22 +702,16 @@ static const config_var_t testing_tor_network_defaults[] = { V(TestingAuthDirTimeToLearnReachability, INTERVAL, "0 minutes"), V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "0 minutes"), V(MinUptimeHidServDirectoryV2, INTERVAL, "0 minutes"), - V(TestingServerDownloadSchedule, CSV_INTERVAL, "0, 0, 0, 5, 10, 15, " - "20, 30, 60"), - V(TestingClientDownloadSchedule, CSV_INTERVAL, "0, 0, 5, 10, 15, 20, " - "30, 60"), - V(TestingServerConsensusDownloadSchedule, CSV_INTERVAL, "0, 0, 5, 10, " - "15, 20, 30, 60"), - V(TestingClientConsensusDownloadSchedule, CSV_INTERVAL, "0, 0, 5, 10, " - "15, 20, 30, 60"), - V(TestingBridgeDownloadSchedule, CSV_INTERVAL, "10, 30, 60"), - V(TestingBridgeBootstrapDownloadSchedule, CSV_INTERVAL, "0, 0, 5, 10, " - "15, 20, 30, 60"), + V(TestingServerDownloadInitialDelay, CSV_INTERVAL, "0"), + V(TestingClientDownloadInitialDelay, CSV_INTERVAL, "0"), + V(TestingServerConsensusDownloadInitialDelay, CSV_INTERVAL, "0"), + V(TestingClientConsensusDownloadInitialDelay, CSV_INTERVAL, "0"), + V(TestingBridgeDownloadInitialDelay, CSV_INTERVAL, "10"), + V(TestingBridgeBootstrapDownloadInitialDelay, CSV_INTERVAL, "0"), V(TestingClientMaxIntervalWithoutRequest, INTERVAL, "5 seconds"), V(TestingDirConnectionMaxStall, INTERVAL, "30 seconds"), V(TestingEnableConnBwEvent, BOOL, "1"), V(TestingEnableCellStatsEvent, BOOL, "1"), - V(TestingEnableTbEmptyEvent, BOOL, "1"), VAR("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_, "1"), V(RendPostPeriod, INTERVAL, "2 minutes"), @@ -747,6 +751,8 @@ static int options_transition_affects_workers( const or_options_t *old_options, const or_options_t *new_options); static int options_transition_affects_descriptor( const or_options_t *old_options, const or_options_t *new_options); +static int options_transition_affects_dirauth_timing( + const or_options_t *old_options, const or_options_t *new_options); static int normalize_nickname_list(config_line_t **normalized_out, const config_line_t *lst, const char *name, char **msg); @@ -905,8 +911,13 @@ set_options(or_options_t *new_val, char **msg) smartlist_free(elements); } - if (old_options != global_options) + if (old_options != global_options) { or_options_free(old_options); + /* If we are here it means we've successfully applied the new options and + * that the global options have been changed to the new values. We'll + * check if we need to remove or add periodic events. */ + periodic_events_on_new_options(global_options); + } return 0; } @@ -1439,9 +1450,9 @@ options_act_reversible(const or_options_t *old_options, char **msg) consider_hibernation(time(NULL)); /* Launch the listeners. (We do this before we setuid, so we can bind to - * ports under 1024.) We don't want to rebind if we're hibernating. If - * networking is disabled, this will close all but the control listeners, - * but disable those. */ + * ports under 1024.) We don't want to rebind if we're hibernating or + * shutting down. If networking is disabled, this will close all but the + * control listeners, but disable those. */ if (!we_are_hibernating()) { if (retry_all_listeners(replaced_listeners, new_listeners, options->DisableNetwork) < 0) { @@ -1545,6 +1556,7 @@ options_act_reversible(const or_options_t *old_options, char **msg) tor_malloc_zero(sizeof(log_severity_list_t)); close_temp_logs(); add_callback_log(severity, control_event_logmsg); + logs_set_pending_callback_callback(control_event_logmsg_pending); control_adjust_event_log_severity(); tor_free(severity); tor_log_update_sigsafe_err_fds(); @@ -1652,8 +1664,7 @@ options_act_reversible(const or_options_t *old_options, char **msg) int options_need_geoip_info(const or_options_t *options, const char **reason_out) { - int bridge_usage = - options->BridgeRelay && options->BridgeRecordUsageByCountry; + int bridge_usage = should_record_bridge_info(options); int routerset_usage = routerset_needs_geoip(options->EntryNodes) || routerset_needs_geoip(options->ExitNodes) || @@ -1743,6 +1754,32 @@ options_transition_affects_guards(const or_options_t *old_options, return 0; } +/** + * Return true if changing the configuration from <b>old</b> to <b>new</b> + * affects the timing of the voting subsystem + */ +static int +options_transition_affects_dirauth_timing(const or_options_t *old_options, + const or_options_t *new_options) +{ + tor_assert(old_options); + tor_assert(new_options); + + if (authdir_mode_v3(old_options) != authdir_mode_v3(new_options)) + return 1; + if (! authdir_mode_v3(new_options)) + return 0; + YES_IF_CHANGED_INT(V3AuthVotingInterval); + YES_IF_CHANGED_INT(V3AuthVoteDelay); + YES_IF_CHANGED_INT(V3AuthDistDelay); + YES_IF_CHANGED_INT(TestingV3AuthInitialVotingInterval); + YES_IF_CHANGED_INT(TestingV3AuthInitialVoteDelay); + YES_IF_CHANGED_INT(TestingV3AuthInitialDistDelay); + YES_IF_CHANGED_INT(TestingV3AuthVotingStartOffset); + + return 0; +} + /** Fetch the active option list, and take actions based on it. All of the * things we do should survive being done repeatedly. If present, * <b>old_options</b> contains the previous value of the options. @@ -1761,7 +1798,6 @@ options_act(const or_options_t *old_options) char *msg=NULL; const int transition_affects_workers = old_options && options_transition_affects_workers(old_options, options); - int old_ewma_enabled; const int transition_affects_guards = old_options && options_transition_affects_guards(old_options, options); @@ -1966,6 +2002,9 @@ options_act(const or_options_t *old_options) finish_daemon(options->DataDirectory); } + /* See whether we need to enable/disable our once-a-second timer. */ + reschedule_per_second_timer(); + /* We want to reinit keys as needed before we do much of anything else: keys are important, and other things can depend on them. */ if (transition_affects_workers || @@ -2035,16 +2074,8 @@ options_act(const or_options_t *old_options) if (accounting_is_enabled(options)) configure_accounting(time(NULL)); - old_ewma_enabled = cell_ewma_enabled(); /* Change the cell EWMA settings */ - cell_ewma_set_scale_factor(options, networkstatus_get_latest_consensus()); - /* If we just enabled ewma, set the cmux policy on all active channels */ - if (cell_ewma_enabled() && !old_ewma_enabled) { - channel_set_cmux_policy_everywhere(&ewma_policy); - } else if (!cell_ewma_enabled() && old_ewma_enabled) { - /* Turn it off everywhere */ - channel_set_cmux_policy_everywhere(NULL); - } + cmux_ewma_set_options(options, networkstatus_get_latest_consensus()); /* Update the BridgePassword's hashed version as needed. We store this as a * digest so that we can do side-channel-proof comparisons on it. @@ -2207,6 +2238,12 @@ options_act(const or_options_t *old_options) options->PerConnBWBurst != old_options->PerConnBWBurst) connection_or_update_token_buckets(get_connection_array(), options); + if (options->BandwidthRate != old_options->BandwidthRate || + options->BandwidthBurst != old_options->BandwidthBurst || + options->RelayBandwidthRate != old_options->RelayBandwidthRate || + options->RelayBandwidthBurst != old_options->RelayBandwidthBurst) + connection_bucket_adjust(options); + if (options->MainloopStats != old_options->MainloopStats) { reset_main_loop_counters(); } @@ -2256,6 +2293,11 @@ options_act(const or_options_t *old_options) } if ((!old_options || !old_options->EntryStatistics) && options->EntryStatistics && !should_record_bridge_info(options)) { + /* If we get here, we've started recording bridge info when we didn't + * do so before. Note that "should_record_bridge_info()" will + * always be false at this point, because of the earlier block + * that cleared EntryStatistics when public_server_mode() was false. + * We're leaving it in as defensive programming. */ if (geoip_is_loaded(AF_INET) || geoip_is_loaded(AF_INET6)) { geoip_entry_stats_init(now); print_notice = 1; @@ -2325,8 +2367,10 @@ options_act(const or_options_t *old_options) /* We may need to reschedule some directory stuff if our status changed. */ if (old_options) { - if (authdir_mode_v3(options) && !authdir_mode_v3(old_options)) - dirvote_recalculate_timing(options, time(NULL)); + if (options_transition_affects_dirauth_timing(old_options, options)) { + voting_schedule_recalculate_timing(options, time(NULL)); + reschedule_dirvote(options); + } if (!bool_eq(directory_fetches_dir_info_early(options), directory_fetches_dir_info_early(old_options)) || !bool_eq(directory_fetches_dir_info_later(options), @@ -3474,12 +3518,14 @@ options_validate(or_options_t *old_options, or_options_t *options, REJECT("Versioning authoritative dir servers must set " "Recommended*Versions."); +#ifdef HAVE_MODULE_DIRAUTH char *t; /* Call these functions to produce warnings only. */ t = format_recommended_version_list(options->RecommendedClientVersions, 1); tor_free(t); t = format_recommended_version_list(options->RecommendedServerVersions, 1); tor_free(t); +#endif if (options->UseEntryGuards) { log_info(LD_CONFIG, "Authoritative directory servers can't set " @@ -3779,6 +3825,11 @@ options_validate(or_options_t *old_options, or_options_t *options, "http://freehaven.net/anonbib/#hs-attack06 for details."); } + if (options->NumPrimaryGuards && options->NumEntryGuards && + options->NumEntryGuards > options->NumPrimaryGuards) { + REJECT("NumEntryGuards must not be greater than NumPrimaryGuards."); + } + if (options->EntryNodes && routerset_is_list(options->EntryNodes) && (routerset_len(options->EntryNodes) == 1) && @@ -3901,15 +3952,6 @@ options_validate(or_options_t *old_options, or_options_t *options, if (options->KeepalivePeriod < 1) REJECT("KeepalivePeriod option must be positive."); - if (options->PortForwarding && options->Sandbox) { - REJECT("PortForwarding is not compatible with Sandbox; at most one can " - "be set"); - } - if (options->PortForwarding && options->NoExec) { - COMPLAIN("Both PortForwarding and NoExec are set; PortForwarding will " - "be ignored."); - } - if (ensure_bandwidth_cap(&options->BandwidthRate, "BandwidthRate", msg) < 0) return -1; @@ -4386,12 +4428,12 @@ options_validate(or_options_t *old_options, or_options_t *options, CHECK_DEFAULT(TestingV3AuthVotingStartOffset); CHECK_DEFAULT(TestingAuthDirTimeToLearnReachability); CHECK_DEFAULT(TestingEstimatedDescriptorPropagationTime); - CHECK_DEFAULT(TestingServerDownloadSchedule); - CHECK_DEFAULT(TestingClientDownloadSchedule); - CHECK_DEFAULT(TestingServerConsensusDownloadSchedule); - CHECK_DEFAULT(TestingClientConsensusDownloadSchedule); - CHECK_DEFAULT(TestingBridgeDownloadSchedule); - CHECK_DEFAULT(TestingBridgeBootstrapDownloadSchedule); + CHECK_DEFAULT(TestingServerDownloadInitialDelay); + CHECK_DEFAULT(TestingClientDownloadInitialDelay); + CHECK_DEFAULT(TestingServerConsensusDownloadInitialDelay); + CHECK_DEFAULT(TestingClientConsensusDownloadInitialDelay); + CHECK_DEFAULT(TestingBridgeDownloadInitialDelay); + CHECK_DEFAULT(TestingBridgeBootstrapDownloadInitialDelay); CHECK_DEFAULT(TestingClientMaxIntervalWithoutRequest); CHECK_DEFAULT(TestingDirConnectionMaxStall); CHECK_DEFAULT(TestingAuthKeyLifetime); @@ -4489,12 +4531,6 @@ options_validate(or_options_t *old_options, or_options_t *options, "Tor networks!"); } - if (options->TestingEnableTbEmptyEvent && - !options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) { - REJECT("TestingEnableTbEmptyEvent may only be changed in testing " - "Tor networks!"); - } - if (options->TestingTorNetwork) { log_warn(LD_CONFIG, "TestingTorNetwork is set. This will make your node " "almost unusable in the public Tor network, and is " @@ -4638,15 +4674,14 @@ have_enough_mem_for_dircache(const or_options_t *options, size_t total_mem, if (options->DirCache) { if (total_mem < DIRCACHE_MIN_MEM_BYTES) { if (options->BridgeRelay) { - *msg = tor_strdup("Running a Bridge with less than " - STRINGIFY(DIRCACHE_MIN_MEM_MB) " MB of memory is not " - "recommended."); + tor_asprintf(msg, "Running a Bridge with less than %d MB of memory " + "is not recommended.", DIRCACHE_MIN_MEM_MB); } else { - *msg = tor_strdup("Being a directory cache (default) with less than " - STRINGIFY(DIRCACHE_MIN_MEM_MB) " MB of memory is not " - "recommended and may consume most of the available " - "resources, consider disabling this functionality by " - "setting the DirCache option to 0."); + tor_asprintf(msg, "Being a directory cache (default) with less than " + "%d MB of memory is not recommended and may consume " + "most of the available resources. Consider disabling " + "this functionality by setting the DirCache option " + "to 0.", DIRCACHE_MIN_MEM_MB); } } } else { @@ -8121,7 +8156,10 @@ getinfo_helper_config(control_connection_t *conn, case CONFIG_TYPE_ISOTIME: type = "Time"; break; case CONFIG_TYPE_ROUTERSET: type = "RouterList"; break; case CONFIG_TYPE_CSV: type = "CommaList"; break; - case CONFIG_TYPE_CSV_INTERVAL: type = "TimeIntervalCommaList"; break; + /* This type accepts more inputs than TimeInterval, but it ignores + * everything after the first entry, so we may as well pretend + * it's a TimeInterval. */ + case CONFIG_TYPE_CSV_INTERVAL: type = "TimeInterval"; break; case CONFIG_TYPE_LINELIST: type = "LineList"; break; case CONFIG_TYPE_LINELIST_S: type = "Dependent"; break; case CONFIG_TYPE_LINELIST_V: type = "Virtual"; break; @@ -8408,3 +8446,17 @@ init_cookie_authentication(const char *fname, const char *header, tor_free(cookie_file_str); return retval; } + +/** + * Return true if any option is set in <b>options</b> to make us behave + * as a client. + */ +int +options_any_client_port_set(const or_options_t *options) +{ + return (options->SocksPort_set || + options->TransPort_set || + options->NATDPort_set || + options->DNSPort_set || + options->HTTPTunnelPort_set); +} diff --git a/src/or/config.h b/src/or/config.h index 1d3c27217e..4b41274434 100644 --- a/src/or/config.h +++ b/src/or/config.h @@ -214,6 +214,9 @@ smartlist_t *get_options_from_transport_options_line(const char *line, const char *transport); smartlist_t *get_options_for_server_transport(const char *transport); +/* Port helper functions. */ +int options_any_client_port_set(const or_options_t *options); + #ifdef CONFIG_PRIVATE #define CL_PORT_NO_STREAM_OPTIONS (1u<<0) diff --git a/src/or/confparse.c b/src/or/confparse.c index 64ed9ee6bb..6bab790945 100644 --- a/src/or/confparse.c +++ b/src/or/confparse.c @@ -162,8 +162,6 @@ config_assign_value(const config_format_t *fmt, void *options, int i, ok; const config_var_t *var; void *lvalue; - int *csv_int; - smartlist_t *csv_str; CONFIG_CHECK(fmt, options); @@ -195,6 +193,30 @@ config_assign_value(const config_format_t *fmt, void *options, *(int *)lvalue = i; break; + case CONFIG_TYPE_CSV_INTERVAL: { + /* We used to have entire smartlists here. But now that all of our + * download schedules use exponential backoff, only the first part + * matters. */ + const char *comma = strchr(c->value, ','); + const char *val = c->value; + char *tmp = NULL; + if (comma) { + tmp = tor_strndup(c->value, comma - c->value); + val = tmp; + } + + i = config_parse_interval(val, &ok); + if (!ok) { + tor_asprintf(msg, + "Interval '%s %s' is malformed or out of bounds.", + c->key, c->value); + return -1; + } + *(int *)lvalue = i; + tor_free(tmp); + break; + } + case CONFIG_TYPE_INTERVAL: { i = config_parse_interval(c->value, &ok); if (!ok) { @@ -298,36 +320,6 @@ config_assign_value(const config_format_t *fmt, void *options, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); break; - case CONFIG_TYPE_CSV_INTERVAL: - if (*(smartlist_t**)lvalue) { - SMARTLIST_FOREACH(*(smartlist_t**)lvalue, int *, cp, tor_free(cp)); - smartlist_clear(*(smartlist_t**)lvalue); - } else { - *(smartlist_t**)lvalue = smartlist_new(); - } - csv_str = smartlist_new(); - smartlist_split_string(csv_str, c->value, ",", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - SMARTLIST_FOREACH_BEGIN(csv_str, char *, str) - { - i = config_parse_interval(str, &ok); - if (!ok) { - tor_asprintf(msg, - "Interval in '%s %s' is malformed or out of bounds.", - c->key, c->value); - SMARTLIST_FOREACH(csv_str, char *, cp, tor_free(cp)); - smartlist_free(csv_str); - return -1; - } - csv_int = tor_malloc_zero(sizeof(int)); - *csv_int = i; - smartlist_add(*(smartlist_t**)lvalue, csv_int); - } - SMARTLIST_FOREACH_END(str); - SMARTLIST_FOREACH(csv_str, char *, cp, tor_free(cp)); - smartlist_free(csv_str); - break; - case CONFIG_TYPE_LINELIST: case CONFIG_TYPE_LINELIST_S: { @@ -528,7 +520,6 @@ config_get_assigned_option(const config_format_t *fmt, const void *options, const config_var_t *var; const void *value; config_line_t *result; - smartlist_t *csv_str; tor_assert(options && key); CONFIG_CHECK(fmt, options); @@ -571,6 +562,7 @@ config_get_assigned_option(const config_format_t *fmt, const void *options, break; } /* fall through */ + case CONFIG_TYPE_CSV_INTERVAL: case CONFIG_TYPE_INTERVAL: case CONFIG_TYPE_MSEC_INTERVAL: case CONFIG_TYPE_UINT: @@ -611,20 +603,6 @@ config_get_assigned_option(const config_format_t *fmt, const void *options, else result->value = tor_strdup(""); break; - case CONFIG_TYPE_CSV_INTERVAL: - if (*(smartlist_t**)value) { - csv_str = smartlist_new(); - SMARTLIST_FOREACH_BEGIN(*(smartlist_t**)value, int *, i) - { - smartlist_add_asprintf(csv_str, "%d", *i); - } - SMARTLIST_FOREACH_END(i); - result->value = smartlist_join_strings(csv_str, ",", 0, NULL); - SMARTLIST_FOREACH(csv_str, char *, cp, tor_free(cp)); - smartlist_free(csv_str); - } else - result->value = tor_strdup(""); - break; case CONFIG_TYPE_OBSOLETE: log_fn(LOG_INFO, LD_CONFIG, "You asked me for the value of an obsolete config option '%s'.", @@ -789,6 +767,7 @@ config_clear(const config_format_t *fmt, void *options, case CONFIG_TYPE_ISOTIME: *(time_t*)lvalue = 0; break; + case CONFIG_TYPE_CSV_INTERVAL: case CONFIG_TYPE_INTERVAL: case CONFIG_TYPE_MSEC_INTERVAL: case CONFIG_TYPE_UINT: @@ -816,13 +795,6 @@ config_clear(const config_format_t *fmt, void *options, *(smartlist_t **)lvalue = NULL; } break; - case CONFIG_TYPE_CSV_INTERVAL: - if (*(smartlist_t**)lvalue) { - SMARTLIST_FOREACH(*(smartlist_t **)lvalue, int *, cp, tor_free(cp)); - smartlist_free(*(smartlist_t **)lvalue); - *(smartlist_t **)lvalue = NULL; - } - break; case CONFIG_TYPE_LINELIST: case CONFIG_TYPE_LINELIST_S: config_free_lines(*(config_line_t **)lvalue); diff --git a/src/or/confparse.h b/src/or/confparse.h index f1f2030343..4b4bf0adb4 100644 --- a/src/or/confparse.h +++ b/src/or/confparse.h @@ -28,7 +28,9 @@ typedef enum config_type_t { * optional whitespace. */ CONFIG_TYPE_CSV_INTERVAL, /**< A list of strings, separated by commas and * optional whitespace, representing intervals in - * seconds, with optional units */ + * seconds, with optional units. We allow + * multiple values here for legacy reasons, but + * ignore every value after the first. */ CONFIG_TYPE_LINELIST, /**< Uninterpreted config lines */ CONFIG_TYPE_LINELIST_S, /**< Uninterpreted, context-sensitive config lines, * mixed with other keywords. */ @@ -62,7 +64,7 @@ typedef union { int *AUTOBOOL; time_t *ISOTIME; smartlist_t **CSV; - smartlist_t **CSV_INTERVAL; + int *CSV_INTERVAL; config_line_t **LINELIST; config_line_t **LINELIST_S; config_line_t **LINELIST_V; diff --git a/src/or/connection.c b/src/or/connection.c index 2a6b10763e..0a2a635096 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -1,4 +1,4 @@ - /* Copyright (c) 2001 Matej Pfajfar. +/* Copyright (c) 2001 Matej Pfajfar. * Copyright (c) 2001-2004, Roger Dingledine. * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. * Copyright (c) 2007-2017, The Tor Project, Inc. */ @@ -76,6 +76,7 @@ #include "connection_edge.h" #include "connection_or.h" #include "control.h" +#include "crypto_util.h" #include "directory.h" #include "dirserv.h" #include "dns.h" @@ -85,6 +86,7 @@ #include "ext_orport.h" #include "geoip.h" #include "main.h" +#include "hibernate.h" #include "hs_common.h" #include "hs_ident.h" #include "nodelist.h" @@ -101,7 +103,6 @@ #include "transports.h" #include "routerparse.h" #include "sandbox.h" -#include "transports.h" #ifdef HAVE_PWD_H #include <pwd.h> @@ -120,8 +121,6 @@ static connection_t *connection_listener_new( static void connection_init(time_t now, connection_t *conn, int type, int socket_family); static int connection_handle_listener_read(connection_t *conn, int new_type); -static int connection_bucket_should_increase(int bucket, - or_connection_t *conn); static int connection_finished_flushing(connection_t *conn); static int connection_flushed_some(connection_t *conn); static int connection_finished_connecting(connection_t *conn); @@ -140,6 +139,8 @@ static const char *proxy_type_to_string(int proxy_type); static int get_proxy_type(void); const tor_addr_t *conn_get_outbound_address(sa_family_t family, const or_options_t *options, unsigned int conn_type); +static void reenable_blocked_connection_init(const or_options_t *options); +static void reenable_blocked_connection_schedule(void); /** The last addresses that our network interface seemed to have been * binding to. We use this as one way to detect when our IP changes. @@ -460,8 +461,8 @@ connection_init(time_t now, connection_t *conn, int type, int socket_family) } conn->timestamp_created = now; - conn->timestamp_lastread = now; - conn->timestamp_lastwritten = now; + conn->timestamp_last_read_allowed = now; + conn->timestamp_last_write_allowed = now; } /** Create a link between <b>conn_a</b> and <b>conn_b</b>. */ @@ -775,8 +776,8 @@ connection_close_immediate(connection_t *conn) connection_unregister_events(conn); /* Prevent the event from getting unblocked. */ - conn->read_blocked_on_bw = - conn->write_blocked_on_bw = 0; + conn->read_blocked_on_bw = 0; + conn->write_blocked_on_bw = 0; if (SOCKET_OK(conn->s)) tor_close_socket(conn->s); @@ -859,7 +860,7 @@ connection_mark_for_close_internal_, (connection_t *conn, /* in case we're going to be held-open-til-flushed, reset * the number of seconds since last successful write, so * we get our whole 15 seconds */ - conn->timestamp_lastwritten = time(NULL); + conn->timestamp_last_write_allowed = time(NULL); } /** Find each connection that has hold_open_until_flushed set to @@ -881,7 +882,7 @@ connection_expire_held_open(void) */ if (conn->hold_open_until_flushed) { tor_assert(conn->marked_for_close); - if (now - conn->timestamp_lastwritten >= 15) { + if (now - conn->timestamp_last_write_allowed >= 15) { int severity; if (conn->type == CONN_TYPE_EXIT || (conn->type == CONN_TYPE_DIR && @@ -1764,13 +1765,13 @@ connection_connect_sockaddr,(connection_t *conn, tor_assert(sa); tor_assert(socket_error); - if (get_options()->DisableNetwork) { - /* We should never even try to connect anyplace if DisableNetwork is set. - * Warn if we do, and refuse to make the connection. + if (net_is_completely_disabled()) { + /* We should never even try to connect anyplace if the network is + * completely shut off. * - * We only check DisableNetwork here, not we_are_hibernating(), since - * we'll still try to fulfill client requests sometimes in the latter case - * (if it is soft hibernation) */ + * (We don't check net_is_disabled() here, since we still sometimes + * want to open connections when we're in soft hibernation.) + */ static ratelim_t disablenet_violated = RATELIM_INIT(30*60); *socket_error = SOCK_ERRNO(ENETUNREACH); log_fn_ratelim(&disablenet_violated, LOG_WARN, LD_BUG, @@ -2820,10 +2821,10 @@ connection_is_rate_limited(connection_t *conn) return 1; } -/** Did either global write bucket run dry last second? If so, - * we are likely to run dry again this second, so be stingy with the - * tokens we just put in. */ -static int write_buckets_empty_last_second = 0; +/** When was either global write bucket last empty? If this was recent, then + * we're probably low on bandwidth, and we should be stingy with our bandwidth + * usage. */ +static time_t write_buckets_last_empty_at = -100; /** How many seconds of no active local circuits will make the * connection revert to the "relayed" bandwidth class? */ @@ -2851,25 +2852,25 @@ connection_counts_as_relayed_traffic(connection_t *conn, time_t now) * write many of them or just a few; and <b>conn_bucket</b> (if * non-negative) provides an upper limit for our answer. */ static ssize_t -connection_bucket_round_robin(int base, int priority, - ssize_t global_bucket, ssize_t conn_bucket) +connection_bucket_get_share(int base, int priority, + ssize_t global_bucket_val, ssize_t conn_bucket) { ssize_t at_most; ssize_t num_bytes_high = (priority ? 32 : 16) * base; ssize_t num_bytes_low = (priority ? 4 : 2) * base; - /* Do a rudimentary round-robin so one circuit can't hog a connection. + /* Do a rudimentary limiting so one circuit can't hog a connection. * Pick at most 32 cells, at least 4 cells if possible, and if we're in * the middle pick 1/8 of the available bandwidth. */ - at_most = global_bucket / 8; + at_most = global_bucket_val / 8; at_most -= (at_most % base); /* round down */ if (at_most > num_bytes_high) /* 16 KB, or 8 KB for low-priority */ at_most = num_bytes_high; else if (at_most < num_bytes_low) /* 2 KB, or 1 KB for low-priority */ at_most = num_bytes_low; - if (at_most > global_bucket) - at_most = global_bucket; + if (at_most > global_bucket_val) + at_most = global_bucket_val; if (conn_bucket >= 0 && at_most > conn_bucket) at_most = conn_bucket; @@ -2885,13 +2886,13 @@ connection_bucket_read_limit(connection_t *conn, time_t now) { int base = RELAY_PAYLOAD_SIZE; int priority = conn->type != CONN_TYPE_DIR; - int conn_bucket = -1; - int global_bucket = global_read_bucket; + ssize_t conn_bucket = -1; + size_t global_bucket_val = token_bucket_rw_get_read(&global_bucket); if (connection_speaks_cells(conn)) { or_connection_t *or_conn = TO_OR_CONN(conn); if (conn->state == OR_CONN_STATE_OPEN) - conn_bucket = or_conn->read_bucket; + conn_bucket = token_bucket_rw_get_read(&or_conn->bucket); base = get_cell_network_size(or_conn->wide_circ_ids); } @@ -2900,12 +2901,13 @@ connection_bucket_read_limit(connection_t *conn, time_t now) return conn_bucket>=0 ? conn_bucket : 1<<14; } - if (connection_counts_as_relayed_traffic(conn, now) && - global_relayed_read_bucket <= global_read_bucket) - global_bucket = global_relayed_read_bucket; + if (connection_counts_as_relayed_traffic(conn, now)) { + size_t relayed = token_bucket_rw_get_read(&global_relayed_bucket); + global_bucket_val = MIN(global_bucket_val, relayed); + } - return connection_bucket_round_robin(base, priority, - global_bucket, conn_bucket); + return connection_bucket_get_share(base, priority, + global_bucket_val, conn_bucket); } /** How many bytes at most can we write onto this connection? */ @@ -2914,8 +2916,8 @@ connection_bucket_write_limit(connection_t *conn, time_t now) { int base = RELAY_PAYLOAD_SIZE; int priority = conn->type != CONN_TYPE_DIR; - int conn_bucket = (int)conn->outbuf_flushlen; - int global_bucket = global_write_bucket; + size_t conn_bucket = conn->outbuf_flushlen; + size_t global_bucket_val = token_bucket_rw_get_write(&global_bucket); if (!connection_is_rate_limited(conn)) { /* be willing to write to local conns even if our buckets are empty */ @@ -2923,22 +2925,21 @@ connection_bucket_write_limit(connection_t *conn, time_t now) } if (connection_speaks_cells(conn)) { - /* use the per-conn write limit if it's lower, but if it's less - * than zero just use zero */ + /* use the per-conn write limit if it's lower */ or_connection_t *or_conn = TO_OR_CONN(conn); if (conn->state == OR_CONN_STATE_OPEN) - if (or_conn->write_bucket < conn_bucket) - conn_bucket = or_conn->write_bucket >= 0 ? - or_conn->write_bucket : 0; + conn_bucket = MIN(conn_bucket, + token_bucket_rw_get_write(&or_conn->bucket)); base = get_cell_network_size(or_conn->wide_circ_ids); } - if (connection_counts_as_relayed_traffic(conn, now) && - global_relayed_write_bucket <= global_write_bucket) - global_bucket = global_relayed_write_bucket; + if (connection_counts_as_relayed_traffic(conn, now)) { + size_t relayed = token_bucket_rw_get_write(&global_relayed_bucket); + global_bucket_val = MIN(global_bucket_val, relayed); + } - return connection_bucket_round_robin(base, priority, - global_bucket, conn_bucket); + return connection_bucket_get_share(base, priority, + global_bucket_val, conn_bucket); } /** Return 1 if the global write buckets are low enough that we @@ -2963,27 +2964,31 @@ connection_bucket_write_limit(connection_t *conn, time_t now) int global_write_bucket_low(connection_t *conn, size_t attempt, int priority) { - int smaller_bucket = global_write_bucket < global_relayed_write_bucket ? - global_write_bucket : global_relayed_write_bucket; + size_t smaller_bucket = + MIN(token_bucket_rw_get_write(&global_bucket), + token_bucket_rw_get_write(&global_relayed_bucket)); if (authdir_mode(get_options()) && priority>1) return 0; /* there's always room to answer v2 if we're an auth dir */ if (!connection_is_rate_limited(conn)) return 0; /* local conns don't get limited */ - if (smaller_bucket < (int)attempt) + if (smaller_bucket < attempt) return 1; /* not enough space no matter the priority */ - if (write_buckets_empty_last_second) - return 1; /* we're already hitting our limits, no more please */ + { + const time_t diff = approx_time() - write_buckets_last_empty_at; + if (diff <= 1) + return 1; /* we're already hitting our limits, no more please */ + } if (priority == 1) { /* old-style v1 query */ /* Could we handle *two* of these requests within the next two seconds? */ const or_options_t *options = get_options(); - int64_t can_write = (int64_t)smaller_bucket + size_t can_write = (size_t) (smaller_bucket + 2*(options->RelayBandwidthRate ? options->RelayBandwidthRate : - options->BandwidthRate); - if (can_write < 2*(int64_t)attempt) + options->BandwidthRate)); + if (can_write < 2*attempt) return 1; } else { /* v2 query */ /* no further constraints yet */ @@ -2991,6 +2996,10 @@ global_write_bucket_low(connection_t *conn, size_t attempt, int priority) return 0; } +/** When did we last tell the accounting subsystem about transmitted + * bandwidth? */ +static time_t last_recorded_accounting_at = 0; + /** Helper: adjusts our bandwidth history and informs the controller as * appropriate, given that we have just read <b>num_read</b> bytes and written * <b>num_written</b> bytes on <b>conn</b>. */ @@ -3021,59 +3030,22 @@ record_num_bytes_transferred_impl(connection_t *conn, } if (conn->type == CONN_TYPE_EXIT) rep_hist_note_exit_bytes(conn->port, num_written, num_read); -} -/** Helper: convert given <b>tvnow</b> time value to milliseconds since - * midnight. */ -static uint32_t -msec_since_midnight(const struct timeval *tvnow) -{ - return (uint32_t)(((tvnow->tv_sec % 86400L) * 1000L) + - ((uint32_t)tvnow->tv_usec / (uint32_t)1000L)); -} + /* Remember these bytes towards statistics. */ + stats_increment_bytes_read_and_written(num_read, num_written); -/** Helper: return the time in milliseconds since <b>last_empty_time</b> - * when a bucket ran empty that previously had <b>tokens_before</b> tokens - * now has <b>tokens_after</b> tokens after refilling at timestamp - * <b>tvnow</b>, capped at <b>milliseconds_elapsed</b> milliseconds since - * last refilling that bucket. Return 0 if the bucket has not been empty - * since the last refill or has not been refilled. */ -uint32_t -bucket_millis_empty(int tokens_before, uint32_t last_empty_time, - int tokens_after, int milliseconds_elapsed, - const struct timeval *tvnow) -{ - uint32_t result = 0, refilled; - if (tokens_before <= 0 && tokens_after > tokens_before) { - refilled = msec_since_midnight(tvnow); - result = (uint32_t)((refilled + 86400L * 1000L - last_empty_time) % - (86400L * 1000L)); - if (result > (uint32_t)milliseconds_elapsed) - result = (uint32_t)milliseconds_elapsed; - } - return result; -} - -/** Check if a bucket which had <b>tokens_before</b> tokens and which got - * <b>tokens_removed</b> tokens removed at timestamp <b>tvnow</b> has run - * out of tokens, and if so, note the milliseconds since midnight in - * <b>timestamp_var</b> for the next TB_EMPTY event. */ -void -connection_buckets_note_empty_ts(uint32_t *timestamp_var, - int tokens_before, size_t tokens_removed, - const struct timeval *tvnow) -{ - if (tokens_before > 0 && (uint32_t)tokens_before <= tokens_removed) - *timestamp_var = msec_since_midnight(tvnow); + /* Remember these bytes towards accounting. */ + if (accounting_is_enabled(get_options())) { + if (now > last_recorded_accounting_at && last_recorded_accounting_at) { + accounting_add_bytes(num_read, num_written, + (int)(now - last_recorded_accounting_at)); + } else { + accounting_add_bytes(num_read, num_written, 0); + } + last_recorded_accounting_at = now; + } } -/** Last time at which the global or relay buckets were emptied in msec - * since midnight. */ -static uint32_t global_relayed_read_emptied = 0, - global_relayed_write_emptied = 0, - global_read_emptied = 0, - global_write_emptied = 0; - /** We just read <b>num_read</b> and wrote <b>num_written</b> bytes * onto <b>conn</b>. Decrement buckets appropriately. */ static void @@ -3098,45 +3070,54 @@ connection_buckets_decrement(connection_t *conn, time_t now, if (!connection_is_rate_limited(conn)) return; /* local IPs are free */ - /* If one or more of our token buckets ran dry just now, note the - * timestamp for TB_EMPTY events. */ - if (get_options()->TestingEnableTbEmptyEvent) { - struct timeval tvnow; - tor_gettimeofday_cached(&tvnow); - if (connection_counts_as_relayed_traffic(conn, now)) { - connection_buckets_note_empty_ts(&global_relayed_read_emptied, - global_relayed_read_bucket, num_read, &tvnow); - connection_buckets_note_empty_ts(&global_relayed_write_emptied, - global_relayed_write_bucket, num_written, &tvnow); - } - connection_buckets_note_empty_ts(&global_read_emptied, - global_read_bucket, num_read, &tvnow); - connection_buckets_note_empty_ts(&global_write_emptied, - global_write_bucket, num_written, &tvnow); - if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN) { - or_connection_t *or_conn = TO_OR_CONN(conn); - connection_buckets_note_empty_ts(&or_conn->read_emptied_time, - or_conn->read_bucket, num_read, &tvnow); - connection_buckets_note_empty_ts(&or_conn->write_emptied_time, - or_conn->write_bucket, num_written, &tvnow); - } + unsigned flags = 0; + if (connection_counts_as_relayed_traffic(conn, now)) { + flags = token_bucket_rw_dec(&global_relayed_bucket, num_read, num_written); } + flags |= token_bucket_rw_dec(&global_bucket, num_read, num_written); - if (connection_counts_as_relayed_traffic(conn, now)) { - global_relayed_read_bucket -= (int)num_read; - global_relayed_write_bucket -= (int)num_written; + if (flags & TB_WRITE) { + write_buckets_last_empty_at = now; } - global_read_bucket -= (int)num_read; - global_write_bucket -= (int)num_written; if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN) { - TO_OR_CONN(conn)->read_bucket -= (int)num_read; - TO_OR_CONN(conn)->write_bucket -= (int)num_written; + or_connection_t *or_conn = TO_OR_CONN(conn); + token_bucket_rw_dec(&or_conn->bucket, num_read, num_written); } } +/** + * Mark <b>conn</b> as needing to stop reading because bandwidth has been + * exhausted. If <b>is_global_bw</b>, it is closing because global bandwidth + * limit has been exhausted. Otherwise, it is closing because its own + * bandwidth limit has been exhausted. + */ +void +connection_read_bw_exhausted(connection_t *conn, bool is_global_bw) +{ + (void)is_global_bw; + conn->read_blocked_on_bw = 1; + connection_stop_reading(conn); + reenable_blocked_connection_schedule(); +} + +/** + * Mark <b>conn</b> as needing to stop reading because write bandwidth has + * been exhausted. If <b>is_global_bw</b>, it is closing because global + * bandwidth limit has been exhausted. Otherwise, it is closing because its + * own bandwidth limit has been exhausted. +*/ +void +connection_write_bw_exhausted(connection_t *conn, bool is_global_bw) +{ + (void)is_global_bw; + conn->write_blocked_on_bw = 1; + connection_stop_writing(conn); + reenable_blocked_connection_schedule(); +} + /** If we have exhausted our global buckets, or the buckets for conn, * stop reading. */ -static void +void connection_consider_empty_read_buckets(connection_t *conn) { const char *reason; @@ -3144,26 +3125,28 @@ connection_consider_empty_read_buckets(connection_t *conn) if (!connection_is_rate_limited(conn)) return; /* Always okay. */ - if (global_read_bucket <= 0) { + int is_global = 1; + + if (token_bucket_rw_get_read(&global_bucket) <= 0) { reason = "global read bucket exhausted. Pausing."; } else if (connection_counts_as_relayed_traffic(conn, approx_time()) && - global_relayed_read_bucket <= 0) { + token_bucket_rw_get_read(&global_relayed_bucket) <= 0) { reason = "global relayed read bucket exhausted. Pausing."; } else if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN && - TO_OR_CONN(conn)->read_bucket <= 0) { + token_bucket_rw_get_read(&TO_OR_CONN(conn)->bucket) <= 0) { reason = "connection read bucket exhausted. Pausing."; + is_global = false; } else return; /* all good, no need to stop it */ LOG_FN_CONN(conn, (LOG_DEBUG, LD_NET, "%s", reason)); - conn->read_blocked_on_bw = 1; - connection_stop_reading(conn); + connection_read_bw_exhausted(conn, is_global); } /** If we have exhausted our global buckets, or the buckets for conn, * stop writing. */ -static void +void connection_consider_empty_write_buckets(connection_t *conn) { const char *reason; @@ -3171,233 +3154,166 @@ connection_consider_empty_write_buckets(connection_t *conn) if (!connection_is_rate_limited(conn)) return; /* Always okay. */ - if (global_write_bucket <= 0) { + bool is_global = true; + if (token_bucket_rw_get_write(&global_bucket) <= 0) { reason = "global write bucket exhausted. Pausing."; } else if (connection_counts_as_relayed_traffic(conn, approx_time()) && - global_relayed_write_bucket <= 0) { + token_bucket_rw_get_write(&global_relayed_bucket) <= 0) { reason = "global relayed write bucket exhausted. Pausing."; } else if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN && - TO_OR_CONN(conn)->write_bucket <= 0) { + token_bucket_rw_get_write(&TO_OR_CONN(conn)->bucket) <= 0) { reason = "connection write bucket exhausted. Pausing."; + is_global = false; } else return; /* all good, no need to stop it */ LOG_FN_CONN(conn, (LOG_DEBUG, LD_NET, "%s", reason)); - conn->write_blocked_on_bw = 1; - connection_stop_writing(conn); + connection_write_bw_exhausted(conn, is_global); } -/** Initialize the global read bucket to options-\>BandwidthBurst. */ +/** Initialize the global buckets to the values configured in the + * options */ void connection_bucket_init(void) { const or_options_t *options = get_options(); - /* start it at max traffic */ - global_read_bucket = (int)options->BandwidthBurst; - global_write_bucket = (int)options->BandwidthBurst; + const uint32_t now_ts = monotime_coarse_get_stamp(); + token_bucket_rw_init(&global_bucket, + (int32_t)options->BandwidthRate, + (int32_t)options->BandwidthBurst, + now_ts); if (options->RelayBandwidthRate) { - global_relayed_read_bucket = (int)options->RelayBandwidthBurst; - global_relayed_write_bucket = (int)options->RelayBandwidthBurst; + token_bucket_rw_init(&global_relayed_bucket, + (int32_t)options->RelayBandwidthRate, + (int32_t)options->RelayBandwidthBurst, + now_ts); } else { - global_relayed_read_bucket = (int)options->BandwidthBurst; - global_relayed_write_bucket = (int)options->BandwidthBurst; + token_bucket_rw_init(&global_relayed_bucket, + (int32_t)options->BandwidthRate, + (int32_t)options->BandwidthBurst, + now_ts); } + + reenable_blocked_connection_init(options); } -/** Refill a single <b>bucket</b> called <b>name</b> with bandwidth rate per - * second <b>rate</b> and bandwidth burst <b>burst</b>, assuming that - * <b>milliseconds_elapsed</b> milliseconds have passed since the last - * call. */ -static void -connection_bucket_refill_helper(int *bucket, int rate, int burst, - int milliseconds_elapsed, - const char *name) +/** Update the global connection bucket settings to a new value. */ +void +connection_bucket_adjust(const or_options_t *options) { - int starting_bucket = *bucket; - if (starting_bucket < burst && milliseconds_elapsed > 0) { - int64_t incr = (((int64_t)rate) * milliseconds_elapsed) / 1000; - if ((burst - starting_bucket) < incr) { - *bucket = burst; /* We would overflow the bucket; just set it to - * the maximum. */ - } else { - *bucket += (int)incr; - if (*bucket > burst || *bucket < starting_bucket) { - /* If we overflow the burst, or underflow our starting bucket, - * cap the bucket value to burst. */ - /* XXXX this might be redundant now, but it doesn't show up - * in profiles. Remove it after analysis. */ - *bucket = burst; - } - } - log_debug(LD_NET,"%s now %d.", name, *bucket); + token_bucket_rw_adjust(&global_bucket, + (int32_t)options->BandwidthRate, + (int32_t)options->BandwidthBurst); + if (options->RelayBandwidthRate) { + token_bucket_rw_adjust(&global_relayed_bucket, + (int32_t)options->RelayBandwidthRate, + (int32_t)options->RelayBandwidthBurst); + } else { + token_bucket_rw_adjust(&global_relayed_bucket, + (int32_t)options->BandwidthRate, + (int32_t)options->BandwidthBurst); } } -/** Time has passed; increment buckets appropriately. */ -void -connection_bucket_refill(int milliseconds_elapsed, time_t now) +/** + * Cached value of the last coarse-timestamp when we refilled the + * global buckets. + */ +static uint32_t last_refilled_global_buckets_ts=0; +/** + * Refill the token buckets for a single connection <b>conn</b>, and the + * global token buckets as appropriate. Requires that <b>now_ts</b> is + * the time in coarse timestamp units. + */ +static void +connection_bucket_refill_single(connection_t *conn, uint32_t now_ts) { - const or_options_t *options = get_options(); - smartlist_t *conns = get_connection_array(); - int bandwidthrate, bandwidthburst, relayrate, relayburst; + /* Note that we only check for equality here: the underlying + * token bucket functions can handle moving backwards in time if they + * need to. */ + if (now_ts != last_refilled_global_buckets_ts) { + token_bucket_rw_refill(&global_bucket, now_ts); + token_bucket_rw_refill(&global_relayed_bucket, now_ts); + last_refilled_global_buckets_ts = now_ts; + } - int prev_global_read = global_read_bucket; - int prev_global_write = global_write_bucket; - int prev_relay_read = global_relayed_read_bucket; - int prev_relay_write = global_relayed_write_bucket; - struct timeval tvnow; /*< Only used if TB_EMPTY events are enabled. */ + if (connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN) { + or_connection_t *or_conn = TO_OR_CONN(conn); + token_bucket_rw_refill(&or_conn->bucket, now_ts); + } +} - bandwidthrate = (int)options->BandwidthRate; - bandwidthburst = (int)options->BandwidthBurst; +/** + * Event to re-enable all connections that were previously blocked on read or + * write. + */ +static mainloop_event_t *reenable_blocked_connections_ev = NULL; - if (options->RelayBandwidthRate) { - relayrate = (int)options->RelayBandwidthRate; - relayburst = (int)options->RelayBandwidthBurst; - } else { - relayrate = bandwidthrate; - relayburst = bandwidthburst; - } - - tor_assert(milliseconds_elapsed >= 0); - - write_buckets_empty_last_second = - global_relayed_write_bucket <= 0 || global_write_bucket <= 0; - - /* refill the global buckets */ - connection_bucket_refill_helper(&global_read_bucket, - bandwidthrate, bandwidthburst, - milliseconds_elapsed, - "global_read_bucket"); - connection_bucket_refill_helper(&global_write_bucket, - bandwidthrate, bandwidthburst, - milliseconds_elapsed, - "global_write_bucket"); - connection_bucket_refill_helper(&global_relayed_read_bucket, - relayrate, relayburst, - milliseconds_elapsed, - "global_relayed_read_bucket"); - connection_bucket_refill_helper(&global_relayed_write_bucket, - relayrate, relayburst, - milliseconds_elapsed, - "global_relayed_write_bucket"); - - /* If buckets were empty before and have now been refilled, tell any - * interested controllers. */ - if (get_options()->TestingEnableTbEmptyEvent) { - uint32_t global_read_empty_time, global_write_empty_time, - relay_read_empty_time, relay_write_empty_time; - tor_gettimeofday_cached(&tvnow); - global_read_empty_time = bucket_millis_empty(prev_global_read, - global_read_emptied, global_read_bucket, - milliseconds_elapsed, &tvnow); - global_write_empty_time = bucket_millis_empty(prev_global_write, - global_write_emptied, global_write_bucket, - milliseconds_elapsed, &tvnow); - control_event_tb_empty("GLOBAL", global_read_empty_time, - global_write_empty_time, milliseconds_elapsed); - relay_read_empty_time = bucket_millis_empty(prev_relay_read, - global_relayed_read_emptied, - global_relayed_read_bucket, - milliseconds_elapsed, &tvnow); - relay_write_empty_time = bucket_millis_empty(prev_relay_write, - global_relayed_write_emptied, - global_relayed_write_bucket, - milliseconds_elapsed, &tvnow); - control_event_tb_empty("RELAY", relay_read_empty_time, - relay_write_empty_time, milliseconds_elapsed); - } - - /* refill the per-connection buckets */ - SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) { - if (connection_speaks_cells(conn)) { - or_connection_t *or_conn = TO_OR_CONN(conn); - int orbandwidthrate = or_conn->bandwidthrate; - int orbandwidthburst = or_conn->bandwidthburst; - - int prev_conn_read = or_conn->read_bucket; - int prev_conn_write = or_conn->write_bucket; - - if (connection_bucket_should_increase(or_conn->read_bucket, or_conn)) { - connection_bucket_refill_helper(&or_conn->read_bucket, - orbandwidthrate, - orbandwidthburst, - milliseconds_elapsed, - "or_conn->read_bucket"); - } - if (connection_bucket_should_increase(or_conn->write_bucket, or_conn)) { - connection_bucket_refill_helper(&or_conn->write_bucket, - orbandwidthrate, - orbandwidthburst, - milliseconds_elapsed, - "or_conn->write_bucket"); - } +/** True iff reenable_blocked_connections_ev is currently scheduled. */ +static int reenable_blocked_connections_is_scheduled = 0; - /* If buckets were empty before and have now been refilled, tell any - * interested controllers. */ - if (get_options()->TestingEnableTbEmptyEvent) { - char *bucket; - uint32_t conn_read_empty_time, conn_write_empty_time; - tor_asprintf(&bucket, "ORCONN ID="U64_FORMAT, - U64_PRINTF_ARG(or_conn->base_.global_identifier)); - conn_read_empty_time = bucket_millis_empty(prev_conn_read, - or_conn->read_emptied_time, - or_conn->read_bucket, - milliseconds_elapsed, &tvnow); - conn_write_empty_time = bucket_millis_empty(prev_conn_write, - or_conn->write_emptied_time, - or_conn->write_bucket, - milliseconds_elapsed, &tvnow); - control_event_tb_empty(bucket, conn_read_empty_time, - conn_write_empty_time, - milliseconds_elapsed); - tor_free(bucket); - } - } +/** Delay after which to run reenable_blocked_connections_ev. */ +static struct timeval reenable_blocked_connections_delay; - if (conn->read_blocked_on_bw == 1 /* marked to turn reading back on now */ - && global_read_bucket > 0 /* and we're allowed to read */ - && (!connection_counts_as_relayed_traffic(conn, now) || - global_relayed_read_bucket > 0) /* even if we're relayed traffic */ - && (!connection_speaks_cells(conn) || - conn->state != OR_CONN_STATE_OPEN || - TO_OR_CONN(conn)->read_bucket > 0)) { - /* and either a non-cell conn or a cell conn with non-empty bucket */ - LOG_FN_CONN(conn, (LOG_DEBUG,LD_NET, - "waking up conn (fd %d) for read", (int)conn->s)); - conn->read_blocked_on_bw = 0; +/** + * Re-enable all connections that were previously blocked on read or write. + * This event is scheduled after enough time has elapsed to be sure + * that the buckets will refill when the connections have something to do. + */ +static void +reenable_blocked_connections_cb(mainloop_event_t *ev, void *arg) +{ + (void)ev; + (void)arg; + SMARTLIST_FOREACH_BEGIN(get_connection_array(), connection_t *, conn) { + if (conn->read_blocked_on_bw == 1) { connection_start_reading(conn); + conn->read_blocked_on_bw = 0; } - - if (conn->write_blocked_on_bw == 1 - && global_write_bucket > 0 /* and we're allowed to write */ - && (!connection_counts_as_relayed_traffic(conn, now) || - global_relayed_write_bucket > 0) /* even if it's relayed traffic */ - && (!connection_speaks_cells(conn) || - conn->state != OR_CONN_STATE_OPEN || - TO_OR_CONN(conn)->write_bucket > 0)) { - LOG_FN_CONN(conn, (LOG_DEBUG,LD_NET, - "waking up conn (fd %d) for write", (int)conn->s)); - conn->write_blocked_on_bw = 0; + if (conn->write_blocked_on_bw == 1) { connection_start_writing(conn); + conn->write_blocked_on_bw = 0; } } SMARTLIST_FOREACH_END(conn); + + reenable_blocked_connections_is_scheduled = 0; } -/** Is the <b>bucket</b> for connection <b>conn</b> low enough that we - * should add another pile of tokens to it? +/** + * Initialize the mainloop event that we use to wake up connections that + * find themselves blocked on bandwidth. */ -static int -connection_bucket_should_increase(int bucket, or_connection_t *conn) +static void +reenable_blocked_connection_init(const or_options_t *options) { - tor_assert(conn); - - if (conn->base_.state != OR_CONN_STATE_OPEN) - return 0; /* only open connections play the rate limiting game */ - if (bucket >= conn->bandwidthburst) - return 0; + if (! reenable_blocked_connections_ev) { + reenable_blocked_connections_ev = + mainloop_event_new(reenable_blocked_connections_cb, NULL); + reenable_blocked_connections_is_scheduled = 0; + } + time_t sec = options->TokenBucketRefillInterval / 1000; + int msec = (options->TokenBucketRefillInterval % 1000); + reenable_blocked_connections_delay.tv_sec = sec; + reenable_blocked_connections_delay.tv_usec = msec * 1000; +} - return 1; +/** + * Called when we have blocked a connection for being low on bandwidth: + * schedule an event to reenable such connections, if it is not already + * scheduled. + */ +static void +reenable_blocked_connection_schedule(void) +{ + if (reenable_blocked_connections_is_scheduled) + return; + if (BUG(reenable_blocked_connections_ev == NULL)) { + reenable_blocked_connection_init(get_options()); + } + mainloop_event_schedule(reenable_blocked_connections_ev, + &reenable_blocked_connections_delay); + reenable_blocked_connections_is_scheduled = 1; } /** Read bytes from conn-\>s and process them. @@ -3418,7 +3334,9 @@ connection_handle_read_impl(connection_t *conn) if (conn->marked_for_close) return 0; /* do nothing */ - conn->timestamp_lastread = approx_time(); + conn->timestamp_last_read_allowed = approx_time(); + + connection_bucket_refill_single(conn, monotime_coarse_get_stamp()); switch (conn->type) { case CONN_TYPE_OR_LISTENER: @@ -3525,8 +3443,7 @@ int connection_handle_read(connection_t *conn) { int res; - - tor_gettimeofday_cache_clear(); + update_current_time(time(NULL)); res = connection_handle_read_impl(conn); return res; } @@ -3678,25 +3595,15 @@ connection_buf_read_from_socket(connection_t *conn, ssize_t *max_to_read, /* change *max_to_read */ *max_to_read = at_most - n_read; - /* Update edge_conn->n_read and ocirc->n_read_circ_bw */ + /* Update edge_conn->n_read */ if (conn->type == CONN_TYPE_AP) { edge_connection_t *edge_conn = TO_EDGE_CONN(conn); - circuit_t *circ = circuit_get_by_edge_conn(edge_conn); - origin_circuit_t *ocirc; /* Check for overflow: */ if (PREDICT_LIKELY(UINT32_MAX - edge_conn->n_read > n_read)) edge_conn->n_read += (int)n_read; else edge_conn->n_read = UINT32_MAX; - - if (circ && CIRCUIT_IS_ORIGIN(circ)) { - ocirc = TO_ORIGIN_CIRCUIT(circ); - if (PREDICT_LIKELY(UINT32_MAX - ocirc->n_read_circ_bw > n_read)) - ocirc->n_read_circ_bw += (int)n_read; - else - ocirc->n_read_circ_bw = UINT32_MAX; - } } /* If CONN_BW events are enabled, update conn->n_read_conn_bw for @@ -3782,7 +3689,6 @@ connection_outbuf_too_full(connection_t *conn) * This should help fix slow upload rates. */ static void - update_send_buffer_size(tor_socket_t sock) { #ifdef _WIN32 @@ -3819,7 +3725,7 @@ update_send_buffer_size(tor_socket_t sock) * when libevent tells us that conn wants to write, or below * from connection_buf_add() when an entire TLS record is ready. * - * Update <b>conn</b>-\>timestamp_lastwritten to now, and call flush_buf + * Update <b>conn</b>-\>timestamp_last_write_allowed to now, and call flush_buf * or flush_buf_tls appropriately. If it succeeds and there are no more * more bytes on <b>conn</b>-\>outbuf, then call connection_finished_flushing * on it too. @@ -3852,7 +3758,9 @@ connection_handle_write_impl(connection_t *conn, int force) return 0; } - conn->timestamp_lastwritten = now; + conn->timestamp_last_write_allowed = now; + + connection_bucket_refill_single(conn, monotime_coarse_get_stamp()); /* Sometimes, "writable" means "connected". */ if (connection_state_is_connecting(conn)) { @@ -3967,8 +3875,7 @@ connection_handle_write_impl(connection_t *conn, int force) /* Make sure to avoid a loop if the receive buckets are empty. */ log_debug(LD_NET,"wanted read."); if (!connection_is_reading(conn)) { - connection_stop_writing(conn); - conn->write_blocked_on_bw = 1; + connection_write_bw_exhausted(conn, true); /* we'll start reading again when we get more tokens in our * read bucket; then we'll start writing again too. */ @@ -4014,22 +3921,12 @@ connection_handle_write_impl(connection_t *conn, int force) if (n_written && conn->type == CONN_TYPE_AP) { edge_connection_t *edge_conn = TO_EDGE_CONN(conn); - circuit_t *circ = circuit_get_by_edge_conn(edge_conn); - origin_circuit_t *ocirc; /* Check for overflow: */ if (PREDICT_LIKELY(UINT32_MAX - edge_conn->n_written > n_written)) edge_conn->n_written += (int)n_written; else edge_conn->n_written = UINT32_MAX; - - if (circ && CIRCUIT_IS_ORIGIN(circ)) { - ocirc = TO_ORIGIN_CIRCUIT(circ); - if (PREDICT_LIKELY(UINT32_MAX - ocirc->n_written_circ_bw > n_written)) - ocirc->n_written_circ_bw += (int)n_written; - else - ocirc->n_written_circ_bw = UINT32_MAX; - } } /* If CONN_BW events are enabled, update conn->n_written_conn_bw for @@ -4089,7 +3986,7 @@ int connection_handle_write(connection_t *conn, int force) { int res; - tor_gettimeofday_cache_clear(); + update_current_time(time(NULL)); conn->in_connection_handle_write = 1; res = connection_handle_write_impl(conn, force); conn->in_connection_handle_write = 0; @@ -4528,8 +4425,6 @@ alloc_http_authenticator(const char *authenticator) static void client_check_address_changed(tor_socket_t sock) { - struct sockaddr_storage out_sockaddr; - socklen_t out_addr_len = (socklen_t) sizeof(out_sockaddr); tor_addr_t out_addr, iface_addr; tor_addr_t **last_interface_ip_ptr; sa_family_t family; @@ -4537,13 +4432,12 @@ client_check_address_changed(tor_socket_t sock) if (!outgoing_addrs) outgoing_addrs = smartlist_new(); - if (getsockname(sock, (struct sockaddr*)&out_sockaddr, &out_addr_len)<0) { + if (tor_addr_from_getsockname(&out_addr, sock) < 0) { int e = tor_socket_errno(sock); log_warn(LD_NET, "getsockname() to check for address change failed: %s", tor_socket_strerror(e)); return; } - tor_addr_from_sockaddr(&out_addr, (struct sockaddr*)&out_sockaddr, NULL); family = tor_addr_family(&out_addr); if (family == AF_INET) @@ -5331,6 +5225,11 @@ connection_free_all(void) tor_free(last_interface_ipv4); tor_free(last_interface_ipv6); + last_recorded_accounting_at = 0; + + mainloop_event_free(reenable_blocked_connections_ev); + reenable_blocked_connections_is_scheduled = 0; + memset(&reenable_blocked_connections_delay, 0, sizeof(struct timeval)); } /** Log a warning, and possibly emit a control event, that <b>received</b> came @@ -5338,10 +5237,10 @@ connection_free_all(void) * that we had more faith in and therefore the warning level should have higher * severity. */ -void -clock_skew_warning(const connection_t *conn, long apparent_skew, int trusted, - log_domain_mask_t domain, const char *received, - const char *source) +MOCK_IMPL(void, +clock_skew_warning, (const connection_t *conn, long apparent_skew, int trusted, + log_domain_mask_t domain, const char *received, + const char *source)) { char dbuf[64]; char *ext_source = NULL, *warn = NULL; diff --git a/src/or/connection.h b/src/or/connection.h index 6bc5a7cfd0..ad3129c9d8 100644 --- a/src/or/connection.h +++ b/src/or/connection.h @@ -122,7 +122,13 @@ void connection_mark_all_noncontrol_connections(void); ssize_t connection_bucket_write_limit(connection_t *conn, time_t now); int global_write_bucket_low(connection_t *conn, size_t attempt, int priority); void connection_bucket_init(void); -void connection_bucket_refill(int seconds_elapsed, time_t now); +void connection_bucket_adjust(const or_options_t *options); +void connection_bucket_refill_all(time_t now, + uint32_t now_ts); +void connection_read_bw_exhausted(connection_t *conn, bool is_global_bw); +void connection_write_bw_exhausted(connection_t *conn, bool is_global_bw); +void connection_consider_empty_read_buckets(connection_t *conn); +void connection_consider_empty_write_buckets(connection_t *conn); int connection_handle_read(connection_t *conn); @@ -248,9 +254,10 @@ void assert_connection_ok(connection_t *conn, time_t now); int connection_or_nonopen_was_started_here(or_connection_t *conn); void connection_dump_buffer_mem_stats(int severity); -void clock_skew_warning(const connection_t *conn, long apparent_skew, - int trusted, log_domain_mask_t domain, - const char *received, const char *source); +MOCK_DECL(void, clock_skew_warning, + (const connection_t *conn, long apparent_skew, int trusted, + log_domain_mask_t domain, const char *received, + const char *source)); /** Check if a connection is on the way out so the OOS handler doesn't try * to kill more than it needs. */ @@ -272,13 +279,6 @@ void connection_check_oos(int n_socks, int failed); STATIC void connection_free_minimal(connection_t *conn); /* Used only by connection.c and test*.c */ -uint32_t bucket_millis_empty(int tokens_before, uint32_t last_empty_time, - int tokens_after, int milliseconds_elapsed, - const struct timeval *tvnow); -void connection_buckets_note_empty_ts(uint32_t *timestamp_var, - int tokens_before, - size_t tokens_removed, - const struct timeval *tvnow); MOCK_DECL(STATIC int,connection_connect_sockaddr, (connection_t *conn, const struct sockaddr *sa, diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index 96b4700e4b..3ae5db9588 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -70,6 +70,7 @@ #include "connection_edge.h" #include "connection_or.h" #include "control.h" +#include "crypto_util.h" #include "dns.h" #include "dnsserv.h" #include "directory.h" @@ -611,6 +612,12 @@ static smartlist_t *pending_entry_connections = NULL; static int untried_pending_connections = 0; +/** + * Mainloop event to tell us to scan for pending connections that can + * be attached. + */ +static mainloop_event_t *attach_pending_entry_connections_ev = NULL; + /** Common code to connection_(ap|exit)_about_to_close. */ static void connection_edge_about_to_close(edge_connection_t *edge_conn) @@ -739,7 +746,7 @@ connection_ap_expire_beginning(void) /* if it's an internal linked connection, don't yell its status. */ severity = (tor_addr_is_null(&base_conn->addr) && !base_conn->port) ? LOG_INFO : LOG_NOTICE; - seconds_idle = (int)( now - base_conn->timestamp_lastread ); + seconds_idle = (int)( now - base_conn->timestamp_last_read_allowed ); seconds_since_born = (int)( now - base_conn->timestamp_created ); if (base_conn->state == AP_CONN_STATE_OPEN) @@ -825,7 +832,7 @@ connection_ap_expire_beginning(void) mark_circuit_unusable_for_new_conns(TO_ORIGIN_CIRCUIT(circ)); /* give our stream another 'cutoff' seconds to try */ - conn->base_.timestamp_lastread += cutoff; + conn->base_.timestamp_last_read_allowed += cutoff; if (entry_conn->num_socks_retries < 250) /* avoid overflow */ entry_conn->num_socks_retries++; /* move it back into 'pending' state, and try to attach. */ @@ -956,6 +963,14 @@ connection_ap_attach_pending(int retry) untried_pending_connections = 0; } +static void +attach_pending_entry_connections_cb(mainloop_event_t *ev, void *arg) +{ + (void)ev; + (void)arg; + connection_ap_attach_pending(0); +} + /** Mark <b>entry_conn</b> as needing to get attached to a circuit. * * And <b>entry_conn</b> must be in AP_CONN_STATE_CIRCUIT_WAIT, @@ -973,9 +988,13 @@ connection_ap_mark_as_pending_circuit_(entry_connection_t *entry_conn, if (conn->marked_for_close) return; - if (PREDICT_UNLIKELY(NULL == pending_entry_connections)) + if (PREDICT_UNLIKELY(NULL == pending_entry_connections)) { pending_entry_connections = smartlist_new(); - + } + if (PREDICT_UNLIKELY(NULL == attach_pending_entry_connections_ev)) { + attach_pending_entry_connections_ev = mainloop_event_postloop_new( + attach_pending_entry_connections_cb, NULL); + } if (PREDICT_UNLIKELY(smartlist_contains(pending_entry_connections, entry_conn))) { log_warn(LD_BUG, "What?? pending_entry_connections already contains %p! " @@ -999,14 +1018,7 @@ connection_ap_mark_as_pending_circuit_(entry_connection_t *entry_conn, untried_pending_connections = 1; smartlist_add(pending_entry_connections, entry_conn); - /* Work-around for bug 19969: we handle pending_entry_connections at - * the end of run_main_loop_once(), but in many cases that function will - * take a very long time, if ever, to finish its call to event_base_loop(). - * - * So the fix is to tell it right now that it ought to finish its loop at - * its next available opportunity. - */ - tell_event_loop_to_run_external_code(); + mainloop_event_activate(attach_pending_entry_connections_ev); } /** Mark <b>entry_conn</b> as no longer waiting for a circuit. */ @@ -1135,7 +1147,7 @@ connection_ap_detach_retriable(entry_connection_t *conn, int reason) { control_event_stream_status(conn, STREAM_EVENT_FAILED_RETRIABLE, reason); - ENTRY_TO_CONN(conn)->timestamp_lastread = time(NULL); + ENTRY_TO_CONN(conn)->timestamp_last_read_allowed = time(NULL); /* Roll back path bias use state so that we probe the circuit * if nothing else succeeds on it */ @@ -3518,16 +3530,24 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ) n_stream->deliver_window = STREAMWINDOW_START; if (circ->purpose == CIRCUIT_PURPOSE_S_REND_JOINED) { + int ret; tor_free(address); /* We handle this circuit and stream in this function for all supported * hidden service version. */ - return handle_hs_exit_conn(circ, n_stream); + ret = handle_hs_exit_conn(circ, n_stream); + + if (ret == 0) { + /* This was a valid cell. Count it as delivered + overhead. */ + circuit_read_valid_data(origin_circ, rh.length); + } + return ret; } tor_strlower(address); n_stream->base_.address = address; n_stream->base_.state = EXIT_CONN_STATE_RESOLVEFAILED; /* default to failed, change in dns_resolve if it turns out not to fail */ + /* If we're hibernating or shutting down, we refuse to open new streams. */ if (we_are_hibernating()) { relay_send_end_cell_from_edge(rh.stream_id, circ, END_STREAM_REASON_HIBERNATING, NULL); @@ -4168,4 +4188,5 @@ connection_edge_free_all(void) untried_pending_connections = 0; smartlist_free(pending_entry_connections); pending_entry_connections = NULL; + mainloop_event_free(attach_pending_entry_connections_ev); } diff --git a/src/or/connection_or.c b/src/or/connection_or.c index 221b8dc8ad..bd5f06bc6a 100644 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@ -39,6 +39,8 @@ #include "connection.h" #include "connection_or.h" #include "control.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "dirserv.h" #include "entrynodes.h" #include "geoip.h" @@ -794,18 +796,10 @@ connection_or_update_token_buckets_helper(or_connection_t *conn, int reset, (int)options->BandwidthBurst, 1, INT32_MAX); } - conn->bandwidthrate = rate; - conn->bandwidthburst = burst; - if (reset) { /* set up the token buckets to be full */ - conn->read_bucket = conn->write_bucket = burst; - return; + token_bucket_rw_adjust(&conn->bucket, rate, burst); + if (reset) { + token_bucket_rw_reset(&conn->bucket, monotime_coarse_get_stamp()); } - /* If the new token bucket is smaller, take out the extra tokens. - * (If it's larger, don't -- the buckets can grow to reach the cap.) */ - if (conn->read_bucket > burst) - conn->read_bucket = burst; - if (conn->write_bucket > burst) - conn->write_bucket = burst; } /** Either our set of relays or our per-conn rate limits have changed. diff --git a/src/or/conscache.c b/src/or/conscache.c index e25ac5f40b..51dc9d621f 100644 --- a/src/or/conscache.c +++ b/src/or/conscache.c @@ -5,6 +5,7 @@ #include "config.h" #include "conscache.h" +#include "crypto_util.h" #include "storagedir.h" #define CCE_MAGIC 0x17162253 diff --git a/src/or/consdiffmgr.c b/src/or/consdiffmgr.c index 02b905a520..323f4f9ca0 100644 --- a/src/or/consdiffmgr.c +++ b/src/or/consdiffmgr.c @@ -99,6 +99,14 @@ static const compress_method_t compress_diffs_with[] = { #endif }; +/** + * Event for rescanning the cache. + */ +static mainloop_event_t *consdiffmgr_rescan_ev = NULL; + +static void consdiffmgr_rescan_cb(mainloop_event_t *ev, void *arg); +static void mark_cdm_cache_dirty(void); + /** How many different methods will we try to use for diff compression? */ STATIC unsigned n_diff_compression_methods(void) @@ -372,7 +380,9 @@ cdm_cache_init(void) } else { consdiffmgr_set_cache_flags(); } - cdm_cache_dirty = 1; + consdiffmgr_rescan_ev = + mainloop_event_postloop_new(consdiffmgr_rescan_cb, NULL); + mark_cdm_cache_dirty(); cdm_cache_loaded = 0; } @@ -1095,6 +1105,24 @@ consdiffmgr_rescan(void) cdm_cache_dirty = 0; } +/** Callback wrapper for consdiffmgr_rescan */ +static void +consdiffmgr_rescan_cb(mainloop_event_t *ev, void *arg) +{ + (void)ev; + (void)arg; + consdiffmgr_rescan(); +} + +/** Mark the cache as dirty, and schedule a rescan event. */ +static void +mark_cdm_cache_dirty(void) +{ + cdm_cache_dirty = 1; + tor_assert(consdiffmgr_rescan_ev); + mainloop_event_activate(consdiffmgr_rescan_ev); +} + /** * Helper: compare two files by their from-valid-after and valid-after labels, * trying to sort in ascending order by from-valid-after (when present) and @@ -1219,6 +1247,7 @@ consdiffmgr_free_all(void) memset(latest_consensus, 0, sizeof(latest_consensus)); consensus_cache_free(cons_diff_cache); cons_diff_cache = NULL; + mainloop_event_free(consdiffmgr_rescan_ev); } /* ===== @@ -1750,7 +1779,7 @@ consensus_compress_worker_replyfn(void *work_) compress_consensus_with, job->out, "consensus"); - cdm_cache_dirty = 1; + mark_cdm_cache_dirty(); unsigned u; consensus_flavor_t f = job->flavor; diff --git a/src/or/control.c b/src/or/control.c index 3f677ba348..1115ef7b1d 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -1,3 +1,4 @@ + /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ @@ -52,6 +53,8 @@ #include "connection_edge.h" #include "connection_or.h" #include "control.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "directory.h" #include "dirserv.h" #include "dnsserv.h" @@ -76,15 +79,13 @@ #include "router.h" #include "routerlist.h" #include "routerparse.h" -#include "shared_random.h" +#include "shared_random_client.h" #ifndef _WIN32 #include <pwd.h> #include <sys/resource.h> #endif -#include <event2/event.h> - #include "crypto_s2k.h" #include "procmon.h" @@ -112,6 +113,10 @@ static int disable_log_messages = 0; #define EVENT_IS_INTERESTING(e) \ (!! (global_event_mask & EVENT_MASK_(e))) +/** Macro: true if any event from the bitfield 'e' is interesting. */ +#define ANY_EVENT_IS_INTERESTING(e) \ + (!! (global_event_mask & (e))) + /** If we're using cookie-type authentication, how long should our cookies be? */ #define AUTHENTICATION_COOKIE_LEN 32 @@ -216,9 +221,10 @@ static void orconn_target_get_name(char *buf, size_t len, static int get_cached_network_liveness(void); static void set_cached_network_liveness(int liveness); -static void flush_queued_events_cb(evutil_socket_t fd, short what, void *arg); +static void flush_queued_events_cb(mainloop_event_t *event, void *arg); static char * download_status_to_string(const download_status_t *dl); +static void control_get_bytes_rw_last_sec(uint64_t *r, uint64_t *w); /** Given a control event code for a message event, return the corresponding * log severity. */ @@ -259,6 +265,8 @@ clear_circ_bw_fields(void) continue; ocirc = TO_ORIGIN_CIRCUIT(circ); ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0; + ocirc->n_overhead_written_circ_bw = ocirc->n_overhead_read_circ_bw = 0; + ocirc->n_delivered_written_circ_bw = ocirc->n_delivered_read_circ_bw = 0; } SMARTLIST_FOREACH_END(circ); } @@ -271,6 +279,7 @@ control_update_global_event_mask(void) smartlist_t *conns = get_connection_array(); event_mask_t old_mask, new_mask; old_mask = global_event_mask; + int any_old_per_sec_events = control_any_per_second_event_enabled(); global_event_mask = 0; SMARTLIST_FOREACH(conns, connection_t *, _conn, @@ -288,10 +297,13 @@ control_update_global_event_mask(void) * we want to hear...*/ control_adjust_event_log_severity(); + /* Macro: true if ev was false before and is true now. */ +#define NEWLY_ENABLED(ev) \ + (! (old_mask & (ev)) && (new_mask & (ev))) + /* ...then, if we've started logging stream or circ bw, clear the * appropriate fields. */ - if (! (old_mask & EVENT_STREAM_BANDWIDTH_USED) && - (new_mask & EVENT_STREAM_BANDWIDTH_USED)) { + if (NEWLY_ENABLED(EVENT_STREAM_BANDWIDTH_USED)) { SMARTLIST_FOREACH(conns, connection_t *, conn, { if (conn->type == CONN_TYPE_AP) { @@ -300,10 +312,18 @@ control_update_global_event_mask(void) } }); } - if (! (old_mask & EVENT_CIRC_BANDWIDTH_USED) && - (new_mask & EVENT_CIRC_BANDWIDTH_USED)) { + if (NEWLY_ENABLED(EVENT_CIRC_BANDWIDTH_USED)) { clear_circ_bw_fields(); } + if (NEWLY_ENABLED(EVENT_BANDWIDTH_USED)) { + uint64_t r, w; + control_get_bytes_rw_last_sec(&r, &w); + } + if (any_old_per_sec_events != control_any_per_second_event_enabled()) { + reschedule_per_second_timer(); + } + +#undef NEWLY_ENABLED } /** Adjust the log severities that result in control_event_logmsg being called @@ -352,6 +372,65 @@ control_event_is_interesting(int event) return EVENT_IS_INTERESTING(event); } +/** Return true if any event that needs to fire once a second is enabled. */ +int +control_any_per_second_event_enabled(void) +{ + return ANY_EVENT_IS_INTERESTING( + EVENT_MASK_(EVENT_BANDWIDTH_USED) | + EVENT_MASK_(EVENT_CELL_STATS) | + EVENT_MASK_(EVENT_CIRC_BANDWIDTH_USED) | + EVENT_MASK_(EVENT_CONN_BW) | + EVENT_MASK_(EVENT_STREAM_BANDWIDTH_USED) + ); +} + +/* The value of 'get_bytes_read()' the previous time that + * control_get_bytes_rw_last_sec() as called. */ +static uint64_t stats_prev_n_read = 0; +/* The value of 'get_bytes_written()' the previous time that + * control_get_bytes_rw_last_sec() as called. */ +static uint64_t stats_prev_n_written = 0; + +/** + * Set <b>n_read</b> and <b>n_written</b> to the total number of bytes read + * and written by Tor since the last call to this function. + * + * Call this only from the main thread. + */ +static void +control_get_bytes_rw_last_sec(uint64_t *n_read, + uint64_t *n_written) +{ + const uint64_t stats_n_bytes_read = get_bytes_read(); + const uint64_t stats_n_bytes_written = get_bytes_written(); + + *n_read = stats_n_bytes_read - stats_prev_n_read; + *n_written = stats_n_bytes_written - stats_prev_n_written; + stats_prev_n_read = stats_n_bytes_read; + stats_prev_n_written = stats_n_bytes_written; +} + +/** + * Run all the controller events (if any) that are scheduled to trigger once + * per second. + */ +void +control_per_second_events(void) +{ + if (!control_any_per_second_event_enabled()) + return; + + uint64_t bytes_read, bytes_written; + control_get_bytes_rw_last_sec(&bytes_read, &bytes_written); + control_event_bandwidth_used((uint32_t)bytes_read,(uint32_t)bytes_written); + + control_event_stream_bandwidth_used(); + control_event_conn_bandwidth_used(); + control_event_circ_bandwidth_used(); + control_event_circuit_cell_stats(); +} + /** Append a NUL-terminated string <b>s</b> to the end of * <b>conn</b>-\>outbuf. */ @@ -691,7 +770,7 @@ static tor_mutex_t *queued_control_events_lock = NULL; /** An event that should fire in order to flush the contents of * queued_control_events. */ -static struct event *flush_queued_events_event = NULL; +static mainloop_event_t *flush_queued_events_event = NULL; void control_initialize_event_queue(void) @@ -703,9 +782,8 @@ control_initialize_event_queue(void) if (flush_queued_events_event == NULL) { struct event_base *b = tor_libevent_get_base(); if (b) { - flush_queued_events_event = tor_event_new(b, - -1, 0, flush_queued_events_cb, - NULL); + flush_queued_events_event = + mainloop_event_new(flush_queued_events_cb, NULL); tor_assert(flush_queued_events_event); } } @@ -781,7 +859,7 @@ queue_control_event_string,(uint16_t event, char *msg)) */ if (activate_event) { tor_assert(flush_queued_events_event); - event_active(flush_queued_events_event, EV_READ, 1); + mainloop_event_activate(flush_queued_events_event); } } @@ -806,6 +884,9 @@ queued_event_free_(queued_event_t *ev) static void queued_events_flush_all(int force) { + /* Make sure that we get all the pending log events, if there are any. */ + flush_pending_log_callbacks(); + if (PREDICT_UNLIKELY(queued_control_events == NULL)) { return; } @@ -863,10 +944,9 @@ queued_events_flush_all(int force) /** Libevent callback: Flushes pending events to controllers that are * interested in them. */ static void -flush_queued_events_cb(evutil_socket_t fd, short what, void *arg) +flush_queued_events_cb(mainloop_event_t *event, void *arg) { - (void) fd; - (void) what; + (void) event; (void) arg; queued_events_flush_all(0); } @@ -1218,7 +1298,6 @@ static const struct control_event_t control_event_table[] = { { EVENT_CONF_CHANGED, "CONF_CHANGED"}, { EVENT_CONN_BW, "CONN_BW" }, { EVENT_CELL_STATS, "CELL_STATS" }, - { EVENT_TB_EMPTY, "TB_EMPTY" }, { EVENT_CIRC_BANDWIDTH_USED, "CIRC_BW" }, { EVENT_TRANSPORT_LAUNCHED, "TRANSPORT_LAUNCHED" }, { EVENT_HS_DESC, "HS_DESC" }, @@ -1784,24 +1863,24 @@ getinfo_helper_misc(control_connection_t *conn, const char *question, } else if (!strcmp(question, "process/pid")) { int myPid = -1; - #ifdef _WIN32 +#ifdef _WIN32 myPid = _getpid(); - #else +#else myPid = getpid(); - #endif +#endif tor_asprintf(answer, "%d", myPid); } else if (!strcmp(question, "process/uid")) { - #ifdef _WIN32 +#ifdef _WIN32 *answer = tor_strdup("-1"); - #else +#else int myUid = geteuid(); tor_asprintf(answer, "%d", myUid); #endif /* defined(_WIN32) */ } else if (!strcmp(question, "process/user")) { - #ifdef _WIN32 +#ifdef _WIN32 *answer = tor_strdup(""); - #else +#else int myUid = geteuid(); const struct passwd *myPwEntry = tor_getpwuid(myUid); @@ -1940,6 +2019,31 @@ getinfo_helper_listeners(control_connection_t *control_conn, return 0; } +/** Implementation helper for GETINFO: answers requests for information about + * the current time in both local and UTF forms. */ +STATIC int +getinfo_helper_current_time(control_connection_t *control_conn, + const char *question, + char **answer, const char **errmsg) +{ + (void)control_conn; + (void)errmsg; + + struct timeval now; + tor_gettimeofday(&now); + char timebuf[ISO_TIME_LEN+1]; + + if (!strcmp(question, "current-time/local")) + format_local_iso_time_nospace(timebuf, (time_t)now.tv_sec); + else if (!strcmp(question, "current-time/utc")) + format_iso_time_nospace(timebuf, (time_t)now.tv_sec); + else + return 0; + + *answer = tor_strdup(timebuf); + return 0; +} + /** Implementation helper for GETINFO: knows the answers for questions about * directory information. */ STATIC int @@ -3082,6 +3186,9 @@ static const getinfo_item_t getinfo_items[] = { DOC("config/defaults", "List of default values for configuration options. " "See also config/names"), + PREFIX("current-time/", current_time, "Current time."), + DOC("current-time/local", "Current time on the local system."), + DOC("current-time/utc", "Current UTC time."), PREFIX("downloads/networkstatus/", downloads, "Download statuses for networkstatus objects"), DOC("downloads/networkstatus/ns", @@ -4499,7 +4606,7 @@ handle_control_add_onion(control_connection_t *conn, const char *body) { smartlist_t *args; - size_t arg_len; + int arg_len; (void) len; /* body is nul-terminated; it's safe to ignore the length */ args = getargs_helper("ADD_ONION", conn, body, 2, -1); if (!args) @@ -4520,7 +4627,7 @@ handle_control_add_onion(control_connection_t *conn, rend_auth_type_t auth_type = REND_NO_AUTH; /* Default to adding an anonymous hidden service if no flag is given */ int non_anonymous = 0; - for (size_t i = 1; i < arg_len; i++) { + for (int i = 1; i < arg_len; i++) { static const char *port_prefix = "Port="; static const char *flags_prefix = "Flags="; static const char *max_s_prefix = "MaxStreams="; @@ -5817,8 +5924,6 @@ control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp, int control_event_stream_bandwidth(edge_connection_t *edge_conn) { - circuit_t *circ; - origin_circuit_t *ocirc; struct timeval now; char tbuf[ISO_TIME_USEC_LEN+1]; if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) { @@ -5834,12 +5939,6 @@ control_event_stream_bandwidth(edge_connection_t *edge_conn) (unsigned long)edge_conn->n_written, tbuf); - circ = circuit_get_by_edge_conn(edge_conn); - if (circ && CIRCUIT_IS_ORIGIN(circ)) { - ocirc = TO_ORIGIN_CIRCUIT(circ); - ocirc->n_read_circ_bw += edge_conn->n_read; - ocirc->n_written_circ_bw += edge_conn->n_written; - } edge_conn->n_written = edge_conn->n_read = 0; } @@ -5902,13 +6001,20 @@ control_event_circ_bandwidth_used(void) tor_gettimeofday(&now); format_iso_time_nospace_usec(tbuf, &now); send_control_event(EVENT_CIRC_BANDWIDTH_USED, - "650 CIRC_BW ID=%d READ=%lu WRITTEN=%lu " - "TIME=%s\r\n", + "650 CIRC_BW ID=%d READ=%lu WRITTEN=%lu TIME=%s " + "DELIVERED_READ=%lu OVERHEAD_READ=%lu " + "DELIVERED_WRITTEN=%lu OVERHEAD_WRITTEN=%lu\r\n", ocirc->global_identifier, (unsigned long)ocirc->n_read_circ_bw, (unsigned long)ocirc->n_written_circ_bw, - tbuf); + tbuf, + (unsigned long)ocirc->n_delivered_read_circ_bw, + (unsigned long)ocirc->n_overhead_read_circ_bw, + (unsigned long)ocirc->n_delivered_written_circ_bw, + (unsigned long)ocirc->n_overhead_written_circ_bw); ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0; + ocirc->n_overhead_written_circ_bw = ocirc->n_overhead_read_circ_bw = 0; + ocirc->n_delivered_written_circ_bw = ocirc->n_delivered_read_circ_bw = 0; } SMARTLIST_FOREACH_END(circ); @@ -6093,28 +6199,6 @@ control_event_circuit_cell_stats(void) return 0; } -/** Tokens in <b>bucket</b> have been refilled: the read bucket was empty - * for <b>read_empty_time</b> millis, the write bucket was empty for - * <b>write_empty_time</b> millis, and buckets were last refilled - * <b>milliseconds_elapsed</b> millis ago. Only emit TB_EMPTY event if - * either read or write bucket have been empty before. */ -int -control_event_tb_empty(const char *bucket, uint32_t read_empty_time, - uint32_t write_empty_time, - int milliseconds_elapsed) -{ - if (get_options()->TestingEnableTbEmptyEvent && - EVENT_IS_INTERESTING(EVENT_TB_EMPTY) && - (read_empty_time > 0 || write_empty_time > 0)) { - send_control_event(EVENT_TB_EMPTY, - "650 TB_EMPTY %s READ=%d WRITTEN=%d " - "LAST=%d\r\n", - bucket, read_empty_time, write_empty_time, - milliseconds_elapsed); - } - return 0; -} - /* about 5 minutes worth. */ #define N_BW_EVENTS_TO_CACHE 300 /* Index into cached_bw_events to next write. */ @@ -6202,7 +6286,7 @@ control_event_logmsg(int severity, uint32_t domain, const char *msg) int event; /* Don't even think of trying to add stuff to a buffer from a cpuworker - * thread. */ + * thread. (See #25987 for plan to fix.) */ if (! in_main_thread()) return; @@ -6248,6 +6332,23 @@ control_event_logmsg(int severity, uint32_t domain, const char *msg) } } +/** + * Logging callback: called when there is a queued pending log callback. + */ +void +control_event_logmsg_pending(void) +{ + if (! in_main_thread()) { + /* We can't handle this case yet, since we're using a + * mainloop_event_t to invoke queued_events_flush_all. We ought to + * use a different mechanism instead: see #25987. + **/ + return; + } + tor_assert(flush_queued_events_event); + mainloop_event_activate(flush_queued_events_event); +} + /** Called whenever we receive new router descriptors: tell any * interested control connections. <b>routers</b> is a list of * routerinfo_t's. @@ -7029,6 +7130,8 @@ control_event_bootstrap_problem(const char *warn, const char *reason, if (bootstrap_problems >= BOOTSTRAP_PROBLEM_THRESHOLD) dowarn = 1; + /* Don't warn about our bootstrapping status if we are hibernating or + * shutting down. */ if (we_are_hibernating()) dowarn = 0; @@ -7598,20 +7701,31 @@ control_event_hs_descriptor_upload_failed(const char *id_digest, void control_free_all(void) { + smartlist_t *queued_events = NULL; + + stats_prev_n_read = stats_prev_n_written = 0; + if (authentication_cookie) /* Free the auth cookie */ tor_free(authentication_cookie); if (detached_onion_services) { /* Free the detached onion services */ SMARTLIST_FOREACH(detached_onion_services, char *, cp, tor_free(cp)); smartlist_free(detached_onion_services); } - if (queued_control_events) { - SMARTLIST_FOREACH(queued_control_events, queued_event_t *, ev, - queued_event_free(ev)); - smartlist_free(queued_control_events); + + if (queued_control_events_lock) { + tor_mutex_acquire(queued_control_events_lock); + flush_queued_event_pending = 0; + queued_events = queued_control_events; queued_control_events = NULL; + tor_mutex_release(queued_control_events_lock); + } + if (queued_events) { + SMARTLIST_FOREACH(queued_events, queued_event_t *, ev, + queued_event_free(ev)); + smartlist_free(queued_events); } if (flush_queued_events_event) { - tor_event_free(flush_queued_events_event); + mainloop_event_free(flush_queued_events_event); flush_queued_events_event = NULL; } bootstrap_percent = BOOTSTRAP_STATUS_UNDEF; @@ -7621,7 +7735,6 @@ control_free_all(void) global_event_mask = 0; disable_log_messages = 0; memset(last_sent_bootstrap_message, 0, sizeof(last_sent_bootstrap_message)); - flush_queued_event_pending = 0; } #ifdef TOR_UNIT_TESTS @@ -7632,3 +7745,4 @@ control_testing_set_global_event_mask(uint64_t mask) global_event_mask = mask; } #endif /* defined(TOR_UNIT_TESTS) */ + diff --git a/src/or/control.h b/src/or/control.h index 28ffeaed86..92cbf866dd 100644 --- a/src/or/control.h +++ b/src/or/control.h @@ -40,6 +40,9 @@ int connection_control_process_inbuf(control_connection_t *conn); #define EVENT_NS 0x000F int control_event_is_interesting(int event); +void control_per_second_events(void); +int control_any_per_second_event_enabled(void); + int control_event_circuit_status(origin_circuit_t *circ, circuit_status_event_t e, int reason); int control_event_circuit_purpose_changed(origin_circuit_t *circ, @@ -59,10 +62,8 @@ int control_event_circ_bandwidth_used(void); int control_event_conn_bandwidth(connection_t *conn); int control_event_conn_bandwidth_used(void); int control_event_circuit_cell_stats(void); -int control_event_tb_empty(const char *bucket, uint32_t read_empty_time, - uint32_t write_empty_time, - int milliseconds_elapsed); void control_event_logmsg(int severity, uint32_t domain, const char *msg); +void control_event_logmsg_pending(void); int control_event_descriptors_changed(smartlist_t *routers); int control_event_address_mapped(const char *from, const char *to, time_t expires, const char *error, @@ -194,7 +195,7 @@ void control_free_all(void); #define EVENT_CONF_CHANGED 0x0019 #define EVENT_CONN_BW 0x001A #define EVENT_CELL_STATS 0x001B -#define EVENT_TB_EMPTY 0x001C +/* UNUSED : 0x001C */ #define EVENT_CIRC_BANDWIDTH_USED 0x001D #define EVENT_TRANSPORT_LAUNCHED 0x0020 #define EVENT_HS_DESC 0x0021 @@ -314,6 +315,10 @@ STATIC int getinfo_helper_dir( control_connection_t *control_conn, const char *question, char **answer, const char **errmsg); +STATIC int getinfo_helper_current_time( + control_connection_t *control_conn, + const char *question, char **answer, + const char **errmsg); #endif /* defined(CONTROL_PRIVATE) */ diff --git a/src/or/cpuworker.c b/src/or/cpuworker.c index 50761dd4d3..15ef6869cf 100644 --- a/src/or/cpuworker.c +++ b/src/or/cpuworker.c @@ -24,14 +24,14 @@ #include "connection_or.h" #include "config.h" #include "cpuworker.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "main.h" #include "onion.h" #include "rephist.h" #include "router.h" #include "workqueue.h" -#include <event2/event.h> - static void queue_pending_tasks(void); typedef struct worker_state_s { @@ -69,22 +69,12 @@ worker_state_free_void(void *arg) static replyqueue_t *replyqueue = NULL; static threadpool_t *threadpool = NULL; -static struct event *reply_event = NULL; static tor_weak_rng_t request_sample_rng = TOR_WEAK_RNG_INIT; static int total_pending_tasks = 0; static int max_pending_tasks = 128; -static void -replyqueue_process_cb(evutil_socket_t sock, short events, void *arg) -{ - replyqueue_t *rq = arg; - (void) sock; - (void) events; - replyqueue_process(rq); -} - /** Initialize the cpuworker subsystem. It is OK to call this more than once * during Tor's lifetime. */ @@ -94,14 +84,6 @@ cpu_init(void) if (!replyqueue) { replyqueue = replyqueue_new(0); } - if (!reply_event) { - reply_event = tor_event_new(tor_libevent_get_base(), - replyqueue_get_socket(replyqueue), - EV_READ|EV_PERSIST, - replyqueue_process_cb, - replyqueue); - event_add(reply_event, NULL); - } if (!threadpool) { /* In our threadpool implementation, half the threads are permissive and @@ -115,7 +97,12 @@ cpu_init(void) worker_state_new, worker_state_free_void, NULL); + + int r = threadpool_register_reply_event(threadpool, NULL); + + tor_assert(r == 0); } + /* Total voodoo. Can we make this more sensible? */ max_pending_tasks = get_num_cpus(get_options()) * 64; crypto_seed_weak_rng(&request_sample_rng); @@ -547,7 +534,7 @@ assign_onionskin_to_cpuworker(or_circuit_t *circ, return 0; } - if (connection_or_digest_is_known_relay(circ->p_chan->identity_digest)) + if (!channel_is_client(circ->p_chan)) rep_hist_note_circuit_handshake_assigned(onionskin->handshake_type); should_time = should_time_request(onionskin->handshake_type); diff --git a/src/or/dircollate.c b/src/or/dirauth/dircollate.c index ce4534ff6c..dec6f75154 100644 --- a/src/or/dircollate.c +++ b/src/or/dirauth/dircollate.c @@ -25,7 +25,6 @@ #include "dircollate.h" #include "dirvote.h" -static void dircollator_collate_by_rsa(dircollator_t *dc); static void dircollator_collate_by_ed25519(dircollator_t *dc); /** Hashtable entry mapping a pair of digests (actually an ed25519 key and an @@ -208,49 +207,18 @@ dircollator_add_vote(dircollator_t *dc, networkstatus_t *v) void dircollator_collate(dircollator_t *dc, int consensus_method) { + (void) consensus_method; + tor_assert(!dc->is_collated); dc->all_rsa_sha1_lst = smartlist_new(); - if (consensus_method < MIN_METHOD_FOR_ED25519_ID_VOTING) - dircollator_collate_by_rsa(dc); - else - dircollator_collate_by_ed25519(dc); + dircollator_collate_by_ed25519(dc); smartlist_sort_digests(dc->all_rsa_sha1_lst); dc->is_collated = 1; } /** - * Collation function for RSA-only consensuses: collate the votes for each - * entry in <b>dc</b> by their RSA keys. - * - * The rule is: - * If an RSA identity key is listed by more than half of the authorities, - * include that identity, and treat all descriptors with that RSA identity - * as describing the same router. - */ -static void -dircollator_collate_by_rsa(dircollator_t *dc) -{ - const int total_authorities = dc->n_authorities; - - DIGESTMAP_FOREACH(dc->by_rsa_sha1, k, vote_routerstatus_t **, vrs_lst) { - int n = 0, i; - for (i = 0; i < dc->n_votes; ++i) { - if (vrs_lst[i] != NULL) - ++n; - } - - if (n <= total_authorities / 2) - continue; - - smartlist_add(dc->all_rsa_sha1_lst, (char *)k); - } DIGESTMAP_FOREACH_END; - - dc->by_collated_rsa_sha1 = dc->by_rsa_sha1; -} - -/** * Collation function for ed25519 consensuses: collate the votes for each * entry in <b>dc</b> by ed25519 key and by RSA key. * diff --git a/src/or/dircollate.h b/src/or/dirauth/dircollate.h index 0584b2fe06..0584b2fe06 100644 --- a/src/or/dircollate.h +++ b/src/or/dirauth/dircollate.h diff --git a/src/or/dirvote.c b/src/or/dirauth/dirvote.c index 7023d4f951..bf05286b3d 100644 --- a/src/or/dirvote.c +++ b/src/or/dirauth/dirvote.c @@ -9,9 +9,10 @@ #include "dircollate.h" #include "directory.h" #include "dirserv.h" -#include "dirvote.h" #include "microdesc.h" #include "networkstatus.h" +#include "nodelist.h" +#include "parsecommon.h" #include "policies.h" #include "protover.h" #include "rephist.h" @@ -21,7 +22,11 @@ #include "routerparse.h" #include "entrynodes.h" /* needed for guardfraction methods */ #include "torcert.h" -#include "shared_random_state.h" +#include "voting_schedule.h" + +#include "dirauth/dirvote.h" +#include "dirauth/mode.h" +#include "dirauth/shared_random_state.h" /** * \file dirvote.c @@ -92,6 +97,30 @@ static int dirvote_compute_consensuses(void); static int dirvote_publish_consensus(void); /* ===== + * Certificate functions + * ===== */ + +/** Allocate and return a new authority_cert_t with the same contents as + * <b>cert</b>. */ +STATIC authority_cert_t * +authority_cert_dup(authority_cert_t *cert) +{ + authority_cert_t *out = tor_malloc(sizeof(authority_cert_t)); + tor_assert(cert); + + memcpy(out, cert, sizeof(authority_cert_t)); + /* Now copy pointed-to things. */ + out->cache_info.signed_descriptor_body = + tor_strndup(cert->cache_info.signed_descriptor_body, + cert->cache_info.signed_descriptor_len); + out->cache_info.saved_location = SAVED_NOWHERE; + out->identity_key = crypto_pk_dup_key(cert->identity_key); + out->signing_key = crypto_pk_dup_key(cert->signing_key); + + return out; +} + +/* ===== * Voting * =====*/ @@ -347,10 +376,73 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key, return status; } +/** Set *<b>timing_out</b> to the intervals at which we would like to vote. + * Note that these aren't the intervals we'll use to vote; they're the ones + * that we'll vote to use. */ +static void +dirvote_get_preferred_voting_intervals(vote_timing_t *timing_out) +{ + const or_options_t *options = get_options(); + + tor_assert(timing_out); + + timing_out->vote_interval = options->V3AuthVotingInterval; + timing_out->n_intervals_valid = options->V3AuthNIntervalsValid; + timing_out->vote_delay = options->V3AuthVoteDelay; + timing_out->dist_delay = options->V3AuthDistDelay; +} + /* ===== * Consensus generation * ===== */ +/** If <b>vrs</b> has a hash made for the consensus method <b>method</b> with + * the digest algorithm <b>alg</b>, decode it and copy it into + * <b>digest256_out</b> and return 0. Otherwise return -1. */ +static int +vote_routerstatus_find_microdesc_hash(char *digest256_out, + const vote_routerstatus_t *vrs, + int method, + digest_algorithm_t alg) +{ + /* XXXX only returns the sha256 method. */ + const vote_microdesc_hash_t *h; + char mstr[64]; + size_t mlen; + char dstr[64]; + + tor_snprintf(mstr, sizeof(mstr), "%d", method); + mlen = strlen(mstr); + tor_snprintf(dstr, sizeof(dstr), " %s=", + crypto_digest_algorithm_get_name(alg)); + + for (h = vrs->microdesc; h; h = h->next) { + const char *cp = h->microdesc_hash_line; + size_t num_len; + /* cp looks like \d+(,\d+)* (digesttype=val )+ . Let's hunt for mstr in + * the first part. */ + while (1) { + num_len = strspn(cp, "1234567890"); + if (num_len == mlen && fast_memeq(mstr, cp, mlen)) { + /* This is the line. */ + char buf[BASE64_DIGEST256_LEN+1]; + /* XXXX ignores extraneous stuff if the digest is too long. This + * seems harmless enough, right? */ + cp = strstr(cp, dstr); + if (!cp) + return -1; + cp += strlen(dstr); + strlcpy(buf, cp, sizeof(buf)); + return digest256_from_base64(digest256_out, buf); + } + if (num_len == 0 || cp[num_len] != ',') + break; + cp += num_len + 1; + } + } + return -1; +} + /** Given a vote <b>vote</b> (not a consensus!), return its associated * networkstatus_voter_info_t. */ static networkstatus_voter_info_t * @@ -363,20 +455,6 @@ get_voter(const networkstatus_t *vote) return smartlist_get(vote->voters, 0); } -/** Return the signature made by <b>voter</b> using the algorithm - * <b>alg</b>, or NULL if none is found. */ -document_signature_t * -voter_get_sig_by_algorithm(const networkstatus_voter_info_t *voter, - digest_algorithm_t alg) -{ - if (!voter->sigs) - return NULL; - SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig, - if (sig->alg == alg) - return sig); - return NULL; -} - /** Temporary structure used in constructing a list of dir-source entries * for a consensus. One of these is generated for every vote, and one more * for every legacy key in each vote. */ @@ -549,12 +627,12 @@ compute_routerstatus_consensus(smartlist_t *votes, int consensus_method, tor_assert(most); - /* If we're producing "a" lines, vote on potential alternative (sets - * of) OR port(s) in the winning routerstatuses. + /* Vote on potential alternative (sets of) OR port(s) in the winning + * routerstatuses. * * XXX prop186 There's at most one alternative OR port (_the_ IPv6 * port) for now. */ - if (consensus_method >= MIN_METHOD_FOR_A_LINES && best_alt_orport_out) { + if (best_alt_orport_out) { smartlist_t *alt_orports = smartlist_new(); const tor_addr_port_t *most_alt_orport = NULL; @@ -664,13 +742,6 @@ compute_consensus_method(smartlist_t *votes) static int consensus_method_is_supported(int method) { - if (method == MIN_METHOD_FOR_ED25519_ID_IN_MD) { - /* This method was broken due to buggy code accidentally left in - * dircollate.c; do not actually use it. - */ - return 0; - } - return (method >= MIN_SUPPORTED_CONSENSUS_METHOD) && (method <= MAX_SUPPORTED_CONSENSUS_METHOD); } @@ -1343,7 +1414,7 @@ compute_nth_protocol_set(int n, int n_voters, const smartlist_t *votes) * behavior, and make the new behavior conditional on a new-enough * consensus_method. **/ -char * +STATIC char * networkstatus_compute_consensus(smartlist_t *votes, int total_authorities, crypto_pk_t *identity_key, @@ -1463,19 +1534,14 @@ networkstatus_compute_consensus(smartlist_t *votes, n_versioning_servers); client_versions = compute_consensus_versions_list(combined_client_versions, n_versioning_clients); - if (consensus_method >= MIN_METHOD_FOR_PACKAGE_LINES) { - packages = compute_consensus_package_lines(votes); - } else { - packages = tor_strdup(""); - } + packages = compute_consensus_package_lines(votes); SMARTLIST_FOREACH(combined_server_versions, char *, cp, tor_free(cp)); SMARTLIST_FOREACH(combined_client_versions, char *, cp, tor_free(cp)); smartlist_free(combined_server_versions); smartlist_free(combined_client_versions); - if (consensus_method >= MIN_METHOD_FOR_ED25519_ID_VOTING) - smartlist_add_strdup(flags, "NoEdConsensus"); + smartlist_add_strdup(flags, "NoEdConsensus"); smartlist_sort_strings(flags); smartlist_uniq_strings(flags); @@ -1524,7 +1590,7 @@ networkstatus_compute_consensus(smartlist_t *votes, tor_free(flaglist); } - if (consensus_method >= MIN_METHOD_FOR_RECOMMENDED_PROTOCOLS) { + { int num_dirauth = get_n_authorities(V3_DIRINFO); int idx; for (idx = 0; idx < 4; ++idx) { @@ -1544,7 +1610,7 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_add_strdup(chunks, "\n"); } - if (consensus_method >= MIN_METHOD_FOR_SHARED_RANDOM) { + { int num_dirauth = get_n_authorities(V3_DIRINFO); /* Default value of this is 2/3 of the total number of authorities. For * instance, if we have 9 dirauth, the default value is 6. The following @@ -1609,7 +1675,7 @@ networkstatus_compute_consensus(smartlist_t *votes, smartlist_free(dir_sources); } - if (consensus_method >= MIN_METHOD_TO_CLIP_UNMEASURED_BW) { + { char *max_unmeasured_param = NULL; /* XXXX Extract this code into a common function. Or don't! see #19011 */ if (params) { @@ -1871,7 +1937,6 @@ networkstatus_compute_consensus(smartlist_t *votes, continue; if (ed_consensus > 0) { - tor_assert(consensus_method >= MIN_METHOD_FOR_ED25519_ID_VOTING); if (ed_consensus <= total_authorities / 2) { log_warn(LD_BUG, "Not enough entries had ed_consensus set; how " "can we have a consensus of %d?", ed_consensus); @@ -1898,10 +1963,8 @@ networkstatus_compute_consensus(smartlist_t *votes, rs_out.published_on = rs->status.published_on; rs_out.dir_port = rs->status.dir_port; rs_out.or_port = rs->status.or_port; - if (consensus_method >= MIN_METHOD_FOR_A_LINES) { - tor_addr_copy(&rs_out.ipv6_addr, &alt_orport.addr); - rs_out.ipv6_orport = alt_orport.port; - } + tor_addr_copy(&rs_out.ipv6_addr, &alt_orport.addr); + rs_out.ipv6_orport = alt_orport.port; rs_out.has_bandwidth = 0; rs_out.has_exitsummary = 0; @@ -1931,8 +1994,7 @@ networkstatus_compute_consensus(smartlist_t *votes, } else if (!strcmp(fl, "Unnamed")) { if (is_unnamed) smartlist_add(chosen_flags, (char*)fl); - } else if (!strcmp(fl, "NoEdConsensus") && - consensus_method >= MIN_METHOD_FOR_ED25519_ID_VOTING) { + } else if (!strcmp(fl, "NoEdConsensus")) { if (ed_consensus <= total_authorities/2) smartlist_add(chosen_flags, (char*)fl); } else { @@ -1959,8 +2021,7 @@ networkstatus_compute_consensus(smartlist_t *votes, /* Starting with consensus method 24, we don't list servers * that are not valid in a consensus. See Proposal 272 */ - if (!is_valid && - consensus_method >= MIN_METHOD_FOR_EXCLUDING_INVALID_NODES) + if (!is_valid) continue; /* Pick the version. */ @@ -1981,8 +2042,7 @@ networkstatus_compute_consensus(smartlist_t *votes, /* If it's a guard and we have enough guardfraction votes, calculate its consensus guardfraction value. */ - if (is_guard && num_guardfraction_inputs > 2 && - consensus_method >= MIN_METHOD_FOR_GUARDFRACTION) { + if (is_guard && num_guardfraction_inputs > 2) { rs_out.has_guardfraction = 1; rs_out.guardfraction_percentage = median_uint32(measured_guardfraction, num_guardfraction_inputs); @@ -1999,8 +2059,7 @@ networkstatus_compute_consensus(smartlist_t *votes, rs_out.has_bandwidth = 1; rs_out.bw_is_unmeasured = 1; rs_out.bandwidth_kb = median_uint32(bandwidths_kb, num_bandwidths); - if (consensus_method >= MIN_METHOD_TO_CLIP_UNMEASURED_BW && - n_authorities_measuring_bandwidth > 2) { + if (n_authorities_measuring_bandwidth > 2) { /* Cap non-measured bandwidths. */ if (rs_out.bandwidth_kb > max_unmeasured_bw_kb) { rs_out.bandwidth_kb = max_unmeasured_bw_kb; @@ -2140,8 +2199,7 @@ networkstatus_compute_consensus(smartlist_t *votes, /* Now the weight line. */ if (rs_out.has_bandwidth) { char *guardfraction_str = NULL; - int unmeasured = rs_out.bw_is_unmeasured && - consensus_method >= MIN_METHOD_TO_CLIP_UNMEASURED_BW; + int unmeasured = rs_out.bw_is_unmeasured; /* If we have guardfraction info, include it in the 'w' line. */ if (rs_out.has_guardfraction) { @@ -2400,7 +2458,7 @@ compute_consensus_package_lines(smartlist_t *votes) * new signature is verifiable.) Return the number of signatures added or * changed, or -1 if the document signed by <b>sigs</b> isn't the same * document as <b>target</b>. */ -int +STATIC int networkstatus_add_detached_signatures(networkstatus_t *target, ns_detached_signatures_t *sigs, const char *source, @@ -2484,7 +2542,7 @@ networkstatus_add_detached_signatures(networkstatus_t *target, continue; } - old_sig = voter_get_sig_by_algorithm(target_voter, sig->alg); + old_sig = networkstatus_get_voter_sig_by_alg(target_voter, sig->alg); /* If the target already has a good signature from this voter, then skip * this one. */ @@ -2592,7 +2650,7 @@ networkstatus_format_signatures(networkstatus_t *consensus, * corresponding to the signatures on <b>consensuses</b>, which must contain * exactly one FLAV_NS consensus, and no more than one consensus for each * other flavor. */ -char * +STATIC char * networkstatus_get_detached_signatures(smartlist_t *consensuses) { smartlist_t *elements; @@ -2697,219 +2755,16 @@ get_detached_signatures_from_pending_consensuses(pending_consensus_t *pending, return signatures; } -/** Release all storage held in <b>s</b>. */ -void -ns_detached_signatures_free_(ns_detached_signatures_t *s) -{ - if (!s) - return; - if (s->signatures) { - STRMAP_FOREACH(s->signatures, flavor, smartlist_t *, sigs) { - SMARTLIST_FOREACH(sigs, document_signature_t *, sig, - document_signature_free(sig)); - smartlist_free(sigs); - } STRMAP_FOREACH_END; - strmap_free(s->signatures, NULL); - strmap_free(s->digests, tor_free_); - } - - tor_free(s); -} - -/* ===== - * Certificate functions - * ===== */ - -/** Allocate and return a new authority_cert_t with the same contents as - * <b>cert</b>. */ -authority_cert_t * -authority_cert_dup(authority_cert_t *cert) -{ - authority_cert_t *out = tor_malloc(sizeof(authority_cert_t)); - tor_assert(cert); - - memcpy(out, cert, sizeof(authority_cert_t)); - /* Now copy pointed-to things. */ - out->cache_info.signed_descriptor_body = - tor_strndup(cert->cache_info.signed_descriptor_body, - cert->cache_info.signed_descriptor_len); - out->cache_info.saved_location = SAVED_NOWHERE; - out->identity_key = crypto_pk_dup_key(cert->identity_key); - out->signing_key = crypto_pk_dup_key(cert->signing_key); - - return out; -} - -/* ===== - * Vote scheduling - * ===== */ - -/** Set *<b>timing_out</b> to the intervals at which we would like to vote. - * Note that these aren't the intervals we'll use to vote; they're the ones - * that we'll vote to use. */ -void -dirvote_get_preferred_voting_intervals(vote_timing_t *timing_out) -{ - const or_options_t *options = get_options(); - - tor_assert(timing_out); - - timing_out->vote_interval = options->V3AuthVotingInterval; - timing_out->n_intervals_valid = options->V3AuthNIntervalsValid; - timing_out->vote_delay = options->V3AuthVoteDelay; - timing_out->dist_delay = options->V3AuthDistDelay; -} - -/** Return the start of the next interval of size <b>interval</b> (in - * seconds) after <b>now</b>, plus <b>offset</b>. Midnight always - * starts a fresh interval, and if the last interval of a day would be - * truncated to less than half its size, it is rolled into the - * previous interval. */ -time_t -dirvote_get_start_of_next_interval(time_t now, int interval, int offset) -{ - struct tm tm; - time_t midnight_today=0; - time_t midnight_tomorrow; - time_t next; - - tor_gmtime_r(&now, &tm); - tm.tm_hour = 0; - tm.tm_min = 0; - tm.tm_sec = 0; - - if (tor_timegm(&tm, &midnight_today) < 0) { - log_warn(LD_BUG, "Ran into an invalid time when trying to find midnight."); - } - midnight_tomorrow = midnight_today + (24*60*60); - - next = midnight_today + ((now-midnight_today)/interval + 1)*interval; - - /* Intervals never cross midnight. */ - if (next > midnight_tomorrow) - next = midnight_tomorrow; - - /* If the interval would only last half as long as it's supposed to, then - * skip over to the next day. */ - if (next + interval/2 > midnight_tomorrow) - next = midnight_tomorrow; - - next += offset; - if (next - interval > now) - next -= interval; - - return next; -} - -/* Populate and return a new voting_schedule_t that can be used to schedule - * voting. The object is allocated on the heap and it's the responsibility of - * the caller to free it. Can't fail. */ -static voting_schedule_t * -get_voting_schedule(const or_options_t *options, time_t now, int severity) -{ - int interval, vote_delay, dist_delay; - time_t start; - time_t end; - networkstatus_t *consensus; - voting_schedule_t *new_voting_schedule; - - new_voting_schedule = tor_malloc_zero(sizeof(voting_schedule_t)); - - consensus = networkstatus_get_live_consensus(now); - - if (consensus) { - interval = (int)( consensus->fresh_until - consensus->valid_after ); - vote_delay = consensus->vote_seconds; - dist_delay = consensus->dist_seconds; - } else { - interval = options->TestingV3AuthInitialVotingInterval; - vote_delay = options->TestingV3AuthInitialVoteDelay; - dist_delay = options->TestingV3AuthInitialDistDelay; - } - - tor_assert(interval > 0); - - if (vote_delay + dist_delay > interval/2) - vote_delay = dist_delay = interval / 4; - - start = new_voting_schedule->interval_starts = - dirvote_get_start_of_next_interval(now,interval, - options->TestingV3AuthVotingStartOffset); - end = dirvote_get_start_of_next_interval(start+1, interval, - options->TestingV3AuthVotingStartOffset); - - tor_assert(end > start); - - new_voting_schedule->fetch_missing_signatures = start - (dist_delay/2); - new_voting_schedule->voting_ends = start - dist_delay; - new_voting_schedule->fetch_missing_votes = - start - dist_delay - (vote_delay/2); - new_voting_schedule->voting_starts = start - dist_delay - vote_delay; - - { - char tbuf[ISO_TIME_LEN+1]; - format_iso_time(tbuf, new_voting_schedule->interval_starts); - tor_log(severity, LD_DIR,"Choosing expected valid-after time as %s: " - "consensus_set=%d, interval=%d", - tbuf, consensus?1:0, interval); - } - - return new_voting_schedule; -} - -#define voting_schedule_free(s) \ - FREE_AND_NULL(voting_schedule_t, voting_schedule_free_, (s)) - -/** Frees a voting_schedule_t. This should be used instead of the generic - * tor_free. */ -static void -voting_schedule_free_(voting_schedule_t *voting_schedule_to_free) -{ - if (!voting_schedule_to_free) - return; - tor_free(voting_schedule_to_free); -} - -static voting_schedule_t voting_schedule; - -/* Using the time <b>now</b>, return the next voting valid-after time. */ +/** + * Entry point: Take whatever voting actions are pending as of <b>now</b>. + * + * Return the time at which the next action should be taken. + */ time_t -dirvote_get_next_valid_after_time(void) -{ - /* This is a safe guard in order to make sure that the voting schedule - * static object is at least initialized. Using this function with a zeroed - * voting schedule can lead to bugs. */ - if (tor_mem_is_zero((const char *) &voting_schedule, - sizeof(voting_schedule))) { - dirvote_recalculate_timing(get_options(), time(NULL)); - voting_schedule.created_on_demand = 1; - } - return voting_schedule.interval_starts; -} - -/** Set voting_schedule to hold the timing for the next vote we should be - * doing. All type of tor do that because HS subsystem needs the timing as - * well to function properly. */ -void -dirvote_recalculate_timing(const or_options_t *options, time_t now) -{ - voting_schedule_t *new_voting_schedule; - - /* get the new voting schedule */ - new_voting_schedule = get_voting_schedule(options, now, LOG_INFO); - tor_assert(new_voting_schedule); - - /* Fill in the global static struct now */ - memcpy(&voting_schedule, new_voting_schedule, sizeof(voting_schedule)); - voting_schedule_free(new_voting_schedule); -} - -/** Entry point: Take whatever voting actions are pending as of <b>now</b>. */ -void dirvote_act(const or_options_t *options, time_t now) { if (!authdir_mode_v3(options)) - return; + return TIME_MAX; tor_assert_nonfatal(voting_schedule.voting_starts); /* If we haven't initialized this object through this codeflow, we need to * recalculate the timings to match our vote. The reason to do that is if we @@ -2923,35 +2778,43 @@ dirvote_act(const or_options_t *options, time_t now) "Mine is %s.", keys, hex_str(c->cache_info.identity_digest, DIGEST_LEN)); tor_free(keys); - dirvote_recalculate_timing(options, now); + voting_schedule_recalculate_timing(options, now); } - if (voting_schedule.voting_starts < now && !voting_schedule.have_voted) { + +#define IF_TIME_FOR_NEXT_ACTION(when_field, done_field) \ + if (! voting_schedule.done_field) { \ + if (voting_schedule.when_field > now) { \ + return voting_schedule.when_field; \ + } else { +#define ENDIF \ + } \ + } + + IF_TIME_FOR_NEXT_ACTION(voting_starts, have_voted) { log_notice(LD_DIR, "Time to vote."); dirvote_perform_vote(); voting_schedule.have_voted = 1; - } - if (voting_schedule.fetch_missing_votes < now && - !voting_schedule.have_fetched_missing_votes) { + } ENDIF + IF_TIME_FOR_NEXT_ACTION(fetch_missing_votes, have_fetched_missing_votes) { log_notice(LD_DIR, "Time to fetch any votes that we're missing."); dirvote_fetch_missing_votes(); voting_schedule.have_fetched_missing_votes = 1; - } - if (voting_schedule.voting_ends < now && - !voting_schedule.have_built_consensus) { + } ENDIF + IF_TIME_FOR_NEXT_ACTION(voting_ends, have_built_consensus) { log_notice(LD_DIR, "Time to compute a consensus."); dirvote_compute_consensuses(); /* XXXX We will want to try again later if we haven't got enough * votes yet. Implement this if it turns out to ever happen. */ voting_schedule.have_built_consensus = 1; - } - if (voting_schedule.fetch_missing_signatures < now && - !voting_schedule.have_fetched_missing_signatures) { + } ENDIF + IF_TIME_FOR_NEXT_ACTION(fetch_missing_signatures, + have_fetched_missing_signatures) { log_notice(LD_DIR, "Time to fetch any signatures that we're missing."); dirvote_fetch_missing_signatures(); voting_schedule.have_fetched_missing_signatures = 1; - } - if (voting_schedule.interval_starts < now && - !voting_schedule.have_published_consensus) { + } ENDIF + IF_TIME_FOR_NEXT_ACTION(interval_starts, + have_published_consensus) { log_notice(LD_DIR, "Time to publish the consensus and discard old votes"); dirvote_publish_consensus(); dirvote_clear_votes(0); @@ -2961,8 +2824,15 @@ dirvote_act(const or_options_t *options, time_t now) networkstatus_get_latest_consensus_by_flavor(FLAV_NS)); /* XXXX We will want to try again later if we haven't got enough * signatures yet. Implement this if it turns out to ever happen. */ - dirvote_recalculate_timing(options, now); - } + voting_schedule_recalculate_timing(options, now); + return voting_schedule.voting_starts; + } ENDIF + + tor_assert_nonfatal_unreached(); + return now + 1; + +#undef ENDIF +#undef IF_TIME_FOR_NEXT_ACTION } /** A vote networkstatus_t and its unparsed body: held around so we can @@ -3826,7 +3696,7 @@ dirvote_get_vote(const char *fp, int flags) /** Construct and return a new microdescriptor from a routerinfo <b>ri</b> * according to <b>consensus_method</b>. **/ -microdesc_t * +STATIC microdesc_t * dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method) { microdesc_t *result = NULL; @@ -3843,8 +3713,7 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method) smartlist_add_asprintf(chunks, "onion-key\n%s", key); - if (consensus_method >= MIN_METHOD_FOR_NTOR_KEY && - ri->onion_curve25519_pkey) { + if (ri->onion_curve25519_pkey) { char kbuf[128]; base64_encode(kbuf, sizeof(kbuf), (const char*)ri->onion_curve25519_pkey->public_key, @@ -3854,8 +3723,7 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method) /* We originally put a lines in the micrdescriptors, but then we worked out * that we needed them in the microdesc consensus. See #20916. */ - if (consensus_method >= MIN_METHOD_FOR_A_LINES && - consensus_method < MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC && + if (consensus_method < MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC && !tor_addr_is_null(&ri->ipv6_addr) && ri->ipv6_orport) smartlist_add_asprintf(chunks, "a %s\n", fmt_addrport(&ri->ipv6_addr, ri->ipv6_orport)); @@ -3866,8 +3734,7 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method) if (summary && strcmp(summary, "reject 1-65535")) smartlist_add_asprintf(chunks, "p %s\n", summary); - if (consensus_method >= MIN_METHOD_FOR_P6_LINES && - ri->ipv6_exit_policy) { + if (ri->ipv6_exit_policy) { /* XXXX+++ This doesn't match proposal 208, which says these should * be taken unchanged from the routerinfo. That's bogosity, IMO: * the proposal should have said to do this instead.*/ @@ -3877,11 +3744,10 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method) tor_free(p6); } - if (consensus_method >= MIN_METHOD_FOR_ID_HASH_IN_MD) { + { char idbuf[ED25519_BASE64_LEN+1]; const char *keytype; - if (consensus_method >= MIN_METHOD_FOR_ED25519_ID_IN_MD && - ri->cache_info.signing_key_cert && + if (ri->cache_info.signing_key_cert && ri->cache_info.signing_key_cert->signing_key_included) { keytype = "ed25519"; ed25519_public_to_base64(idbuf, @@ -3925,7 +3791,7 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method) * in a consensus vote document. Write it into the <b>out_len</b>-byte buffer * in <b>out</b>. Return -1 on failure and the number of characters written * on success. */ -ssize_t +static ssize_t dirvote_format_microdesc_vote_line(char *out_buf, size_t out_buf_len, const microdesc_t *md, int consensus_method_low, @@ -3959,13 +3825,7 @@ static const struct consensus_method_range_t { int low; int high; } microdesc_consensus_methods[] = { - {MIN_SUPPORTED_CONSENSUS_METHOD, MIN_METHOD_FOR_A_LINES - 1}, - {MIN_METHOD_FOR_A_LINES, MIN_METHOD_FOR_P6_LINES - 1}, - {MIN_METHOD_FOR_P6_LINES, MIN_METHOD_FOR_NTOR_KEY - 1}, - {MIN_METHOD_FOR_NTOR_KEY, MIN_METHOD_FOR_ID_HASH_IN_MD - 1}, - {MIN_METHOD_FOR_ID_HASH_IN_MD, MIN_METHOD_FOR_ED25519_ID_IN_MD - 1}, - {MIN_METHOD_FOR_ED25519_ID_IN_MD, - MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC - 1}, + {MIN_SUPPORTED_CONSENSUS_METHOD, MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC - 1}, {MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC, MAX_SUPPORTED_CONSENSUS_METHOD}, {-1, -1} }; @@ -4039,49 +3899,692 @@ dirvote_format_all_microdesc_vote_lines(const routerinfo_t *ri, time_t now, return result; } -/** If <b>vrs</b> has a hash made for the consensus method <b>method</b> with - * the digest algorithm <b>alg</b>, decode it and copy it into - * <b>digest256_out</b> and return 0. Otherwise return -1. */ -int -vote_routerstatus_find_microdesc_hash(char *digest256_out, - const vote_routerstatus_t *vrs, - int method, - digest_algorithm_t alg) +/** Parse and extract all SR commits from <b>tokens</b> and place them in + * <b>ns</b>. */ +static void +extract_shared_random_commits(networkstatus_t *ns, const smartlist_t *tokens) { - /* XXXX only returns the sha256 method. */ - const vote_microdesc_hash_t *h; - char mstr[64]; - size_t mlen; - char dstr[64]; + smartlist_t *chunks = NULL; - tor_snprintf(mstr, sizeof(mstr), "%d", method); - mlen = strlen(mstr); - tor_snprintf(dstr, sizeof(dstr), " %s=", - crypto_digest_algorithm_get_name(alg)); + tor_assert(ns); + tor_assert(tokens); + /* Commits are only present in a vote. */ + tor_assert(ns->type == NS_TYPE_VOTE); - for (h = vrs->microdesc; h; h = h->next) { - const char *cp = h->microdesc_hash_line; - size_t num_len; - /* cp looks like \d+(,\d+)* (digesttype=val )+ . Let's hunt for mstr in - * the first part. */ - while (1) { - num_len = strspn(cp, "1234567890"); - if (num_len == mlen && fast_memeq(mstr, cp, mlen)) { - /* This is the line. */ - char buf[BASE64_DIGEST256_LEN+1]; - /* XXXX ignores extraneous stuff if the digest is too long. This - * seems harmless enough, right? */ - cp = strstr(cp, dstr); - if (!cp) - return -1; - cp += strlen(dstr); - strlcpy(buf, cp, sizeof(buf)); - return digest256_from_base64(digest256_out, buf); + ns->sr_info.commits = smartlist_new(); + + smartlist_t *commits = find_all_by_keyword(tokens, K_COMMIT); + /* It's normal that a vote might contain no commits even if it participates + * in the SR protocol. Don't treat it as an error. */ + if (commits == NULL) { + goto end; + } + + /* Parse the commit. We do NO validation of number of arguments or ordering + * for forward compatibility, it's the parse commit job to inform us if it's + * supported or not. */ + chunks = smartlist_new(); + SMARTLIST_FOREACH_BEGIN(commits, directory_token_t *, tok) { + /* Extract all arguments and put them in the chunks list. */ + for (int i = 0; i < tok->n_args; i++) { + smartlist_add(chunks, tok->args[i]); + } + sr_commit_t *commit = sr_parse_commit(chunks); + smartlist_clear(chunks); + if (commit == NULL) { + /* Get voter identity so we can warn that this dirauth vote contains + * commit we can't parse. */ + networkstatus_voter_info_t *voter = smartlist_get(ns->voters, 0); + tor_assert(voter); + log_warn(LD_DIR, "SR: Unable to parse commit %s from vote of voter %s.", + escaped(tok->object_body), + hex_str(voter->identity_digest, + sizeof(voter->identity_digest))); + /* Commitment couldn't be parsed. Continue onto the next commit because + * this one could be unsupported for instance. */ + continue; + } + /* Add newly created commit object to the vote. */ + smartlist_add(ns->sr_info.commits, commit); + } SMARTLIST_FOREACH_END(tok); + + end: + smartlist_free(chunks); + smartlist_free(commits); +} + +/* Using the given directory tokens in tokens, parse the shared random commits + * and put them in the given vote document ns. + * + * This also sets the SR participation flag if present in the vote. */ +void +dirvote_parse_sr_commits(networkstatus_t *ns, const smartlist_t *tokens) +{ + /* Does this authority participates in the SR protocol? */ + directory_token_t *tok = find_opt_by_keyword(tokens, K_SR_FLAG); + if (tok) { + ns->sr_info.participate = 1; + /* Get the SR commitments and reveals from the vote. */ + extract_shared_random_commits(ns, tokens); + } +} + +/* For the given vote, free the shared random commits if any. */ +void +dirvote_clear_commits(networkstatus_t *ns) +{ + tor_assert(ns->type == NS_TYPE_VOTE); + + if (ns->sr_info.commits) { + SMARTLIST_FOREACH(ns->sr_info.commits, sr_commit_t *, c, + sr_commit_free(c)); + smartlist_free(ns->sr_info.commits); + } +} + +/* The given url is the /tor/status-vote GET directory request. Populates the + * items list with strings that we can compress on the fly and dir_items with + * cached_dir_t objects that have a precompressed deflated version. */ +void +dirvote_dirreq_get_status_vote(const char *url, smartlist_t *items, + smartlist_t *dir_items) +{ + int current; + + url += strlen("/tor/status-vote/"); + current = !strcmpstart(url, "current/"); + url = strchr(url, '/'); + tor_assert(url); + ++url; + if (!strcmp(url, "consensus")) { + const char *item; + tor_assert(!current); /* we handle current consensus specially above, + * since it wants to be spooled. */ + if ((item = dirvote_get_pending_consensus(FLAV_NS))) + smartlist_add(items, (char*)item); + } else if (!current && !strcmp(url, "consensus-signatures")) { + /* XXXX the spec says that we should implement + * current/consensus-signatures too. It doesn't seem to be needed, + * though. */ + const char *item; + if ((item=dirvote_get_pending_detached_signatures())) + smartlist_add(items, (char*)item); + } else if (!strcmp(url, "authority")) { + const cached_dir_t *d; + int flags = DGV_BY_ID | + (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING); + if ((d=dirvote_get_vote(NULL, flags))) + smartlist_add(dir_items, (cached_dir_t*)d); + } else { + const cached_dir_t *d; + smartlist_t *fps = smartlist_new(); + int flags; + if (!strcmpstart(url, "d/")) { + url += 2; + flags = DGV_INCLUDE_PENDING | DGV_INCLUDE_PREVIOUS; + } else { + flags = DGV_BY_ID | + (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING); + } + dir_split_resource_into_fingerprints(url, fps, NULL, + DSR_HEX|DSR_SORT_UNIQ); + SMARTLIST_FOREACH(fps, char *, fp, { + if ((d = dirvote_get_vote(fp, flags))) + smartlist_add(dir_items, (cached_dir_t*)d); + tor_free(fp); + }); + smartlist_free(fps); + } +} + +/** Get the best estimate of a router's bandwidth for dirauth purposes, + * preferring measured to advertised values if available. */ +static uint32_t +dirserv_get_bandwidth_for_router_kb(const routerinfo_t *ri) +{ + uint32_t bw_kb = 0; + /* + * Yeah, measured bandwidths in measured_bw_line_t are (implicitly + * signed) longs and the ones router_get_advertised_bandwidth() returns + * are uint32_t. + */ + long mbw_kb = 0; + + if (ri) { + /* + * * First try to see if we have a measured bandwidth; don't bother with + * as_of_out here, on the theory that a stale measured bandwidth is still + * better to trust than an advertised one. + */ + if (dirserv_query_measured_bw_cache_kb(ri->cache_info.identity_digest, + &mbw_kb, NULL)) { + /* Got one! */ + bw_kb = (uint32_t)mbw_kb; + } else { + /* If not, fall back to advertised */ + bw_kb = router_get_advertised_bandwidth(ri) / 1000; + } + } + + return bw_kb; +} + +/** Helper for sorting: compares two routerinfos first by address, and then by + * descending order of "usefulness". (An authority is more useful than a + * non-authority; a running router is more useful than a non-running router; + * and a router with more bandwidth is more useful than one with less.) + **/ +static int +compare_routerinfo_by_ip_and_bw_(const void **a, const void **b) +{ + routerinfo_t *first = *(routerinfo_t **)a, *second = *(routerinfo_t **)b; + int first_is_auth, second_is_auth; + uint32_t bw_kb_first, bw_kb_second; + const node_t *node_first, *node_second; + int first_is_running, second_is_running; + + /* we return -1 if first should appear before second... that is, + * if first is a better router. */ + if (first->addr < second->addr) + return -1; + else if (first->addr > second->addr) + return 1; + + /* Potentially, this next bit could cause k n lg n memeq calls. But in + * reality, we will almost never get here, since addresses will usually be + * different. */ + + first_is_auth = + router_digest_is_trusted_dir(first->cache_info.identity_digest); + second_is_auth = + router_digest_is_trusted_dir(second->cache_info.identity_digest); + + if (first_is_auth && !second_is_auth) + return -1; + else if (!first_is_auth && second_is_auth) + return 1; + + node_first = node_get_by_id(first->cache_info.identity_digest); + node_second = node_get_by_id(second->cache_info.identity_digest); + first_is_running = node_first && node_first->is_running; + second_is_running = node_second && node_second->is_running; + + if (first_is_running && !second_is_running) + return -1; + else if (!first_is_running && second_is_running) + return 1; + + bw_kb_first = dirserv_get_bandwidth_for_router_kb(first); + bw_kb_second = dirserv_get_bandwidth_for_router_kb(second); + + if (bw_kb_first > bw_kb_second) + return -1; + else if (bw_kb_first < bw_kb_second) + return 1; + + /* They're equal! Compare by identity digest, so there's a + * deterministic order and we avoid flapping. */ + return fast_memcmp(first->cache_info.identity_digest, + second->cache_info.identity_digest, + DIGEST_LEN); +} + +/** Given a list of routerinfo_t in <b>routers</b>, return a new digestmap_t + * whose keys are the identity digests of those routers that we're going to + * exclude for Sybil-like appearance. */ +static digestmap_t * +get_possible_sybil_list(const smartlist_t *routers) +{ + const or_options_t *options = get_options(); + digestmap_t *omit_as_sybil; + smartlist_t *routers_by_ip = smartlist_new(); + uint32_t last_addr; + int addr_count; + /* Allow at most this number of Tor servers on a single IP address, ... */ + int max_with_same_addr = options->AuthDirMaxServersPerAddr; + if (max_with_same_addr <= 0) + max_with_same_addr = INT_MAX; + + smartlist_add_all(routers_by_ip, routers); + smartlist_sort(routers_by_ip, compare_routerinfo_by_ip_and_bw_); + omit_as_sybil = digestmap_new(); + + last_addr = 0; + addr_count = 0; + SMARTLIST_FOREACH_BEGIN(routers_by_ip, routerinfo_t *, ri) { + if (last_addr != ri->addr) { + last_addr = ri->addr; + addr_count = 1; + } else if (++addr_count > max_with_same_addr) { + digestmap_set(omit_as_sybil, ri->cache_info.identity_digest, ri); + } + } SMARTLIST_FOREACH_END(ri); + + smartlist_free(routers_by_ip); + return omit_as_sybil; +} + +/** Given a platform string as in a routerinfo_t (possibly null), return a + * newly allocated version string for a networkstatus document, or NULL if the + * platform doesn't give a Tor version. */ +static char * +version_from_platform(const char *platform) +{ + if (platform && !strcmpstart(platform, "Tor ")) { + const char *eos = find_whitespace(platform+4); + if (eos && !strcmpstart(eos, " (r")) { + /* XXXX Unify this logic with the other version extraction + * logic in routerparse.c. */ + eos = find_whitespace(eos+1); + } + if (eos) { + return tor_strndup(platform, eos-platform); + } + } + return NULL; +} + +/** Given a (possibly empty) list of config_line_t, each line of which contains + * a list of comma-separated version numbers surrounded by optional space, + * allocate and return a new string containing the version numbers, in order, + * separated by commas. Used to generate Recommended(Client|Server)?Versions + */ +char * +format_recommended_version_list(const config_line_t *ln, int warn) +{ + smartlist_t *versions; + char *result; + versions = smartlist_new(); + for ( ; ln; ln = ln->next) { + smartlist_split_string(versions, ln->value, ",", + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + } + + /* Handle the case where a dirauth operator has accidentally made some + * versions space-separated instead of comma-separated. */ + smartlist_t *more_versions = smartlist_new(); + SMARTLIST_FOREACH_BEGIN(versions, char *, v) { + if (strchr(v, ' ')) { + if (warn) + log_warn(LD_DIRSERV, "Unexpected space in versions list member %s. " + "(These are supposed to be comma-separated; I'll pretend you " + "used commas instead.)", escaped(v)); + SMARTLIST_DEL_CURRENT(versions, v); + smartlist_split_string(more_versions, v, NULL, + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + tor_free(v); + } + } SMARTLIST_FOREACH_END(v); + smartlist_add_all(versions, more_versions); + smartlist_free(more_versions); + + /* Check to make sure everything looks like a version. */ + if (warn) { + SMARTLIST_FOREACH_BEGIN(versions, const char *, v) { + tor_version_t ver; + if (tor_version_parse(v, &ver) < 0) { + log_warn(LD_DIRSERV, "Recommended version %s does not look valid. " + " (I'll include it anyway, since you told me to.)", + escaped(v)); } - if (num_len == 0 || cp[num_len] != ',') - break; - cp += num_len + 1; + } SMARTLIST_FOREACH_END(v); + } + + sort_version_list(versions, 1); + result = smartlist_join_strings(versions,",",0,NULL); + SMARTLIST_FOREACH(versions,char *,s,tor_free(s)); + smartlist_free(versions); + return result; +} + +/** If there are entries in <b>routers</b> with exactly the same ed25519 keys, + * remove the older one. If they are exactly the same age, remove the one + * with the greater descriptor digest. May alter the order of the list. */ +static void +routers_make_ed_keys_unique(smartlist_t *routers) +{ + routerinfo_t *ri2; + digest256map_t *by_ed_key = digest256map_new(); + + SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { + ri->omit_from_vote = 0; + if (ri->cache_info.signing_key_cert == NULL) + continue; /* No ed key */ + const uint8_t *pk = ri->cache_info.signing_key_cert->signing_key.pubkey; + if ((ri2 = digest256map_get(by_ed_key, pk))) { + /* Duplicate; must omit one. Set the omit_from_vote flag in whichever + * one has the earlier published_on. */ + const time_t ri_pub = ri->cache_info.published_on; + const time_t ri2_pub = ri2->cache_info.published_on; + if (ri2_pub < ri_pub || + (ri2_pub == ri_pub && + fast_memcmp(ri->cache_info.signed_descriptor_digest, + ri2->cache_info.signed_descriptor_digest,DIGEST_LEN)<0)) { + digest256map_set(by_ed_key, pk, ri); + ri2->omit_from_vote = 1; + } else { + ri->omit_from_vote = 1; + } + } else { + /* Add to map */ + digest256map_set(by_ed_key, pk, ri); } + } SMARTLIST_FOREACH_END(ri); + + digest256map_free(by_ed_key, NULL); + + /* Now remove every router where the omit_from_vote flag got set. */ + SMARTLIST_FOREACH_BEGIN(routers, const routerinfo_t *, ri) { + if (ri->omit_from_vote) { + SMARTLIST_DEL_CURRENT(routers, ri); + } + } SMARTLIST_FOREACH_END(ri); +} + +/** Routerstatus <b>rs</b> is part of a group of routers that are on + * too narrow an IP-space. Clear out its flags since we don't want it be used + * because of its Sybil-like appearance. + * + * Leave its BadExit flag alone though, since if we think it's a bad exit, + * we want to vote that way in case all the other authorities are voting + * Running and Exit. + */ +static void +clear_status_flags_on_sybil(routerstatus_t *rs) +{ + rs->is_authority = rs->is_exit = rs->is_stable = rs->is_fast = + rs->is_flagged_running = rs->is_named = rs->is_valid = + rs->is_hs_dir = rs->is_v2_dir = rs->is_possible_guard = 0; + /* FFFF we might want some mechanism to check later on if we + * missed zeroing any flags: it's easy to add a new flag but + * forget to add it to this clause. */ +} + +/** Return a new networkstatus_t* containing our current opinion. (For v3 + * authorities) */ +networkstatus_t * +dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, + authority_cert_t *cert) +{ + const or_options_t *options = get_options(); + networkstatus_t *v3_out = NULL; + uint32_t addr; + char *hostname = NULL, *client_versions = NULL, *server_versions = NULL; + const char *contact; + smartlist_t *routers, *routerstatuses; + char identity_digest[DIGEST_LEN]; + char signing_key_digest[DIGEST_LEN]; + int listbadexits = options->AuthDirListBadExits; + routerlist_t *rl = router_get_routerlist(); + time_t now = time(NULL); + time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH; + networkstatus_voter_info_t *voter = NULL; + vote_timing_t timing; + digestmap_t *omit_as_sybil = NULL; + const int vote_on_reachability = running_long_enough_to_decide_unreachable(); + smartlist_t *microdescriptors = NULL; + + tor_assert(private_key); + tor_assert(cert); + + if (crypto_pk_get_digest(private_key, signing_key_digest)<0) { + log_err(LD_BUG, "Error computing signing key digest"); + return NULL; } - return -1; + if (crypto_pk_get_digest(cert->identity_key, identity_digest)<0) { + log_err(LD_BUG, "Error computing identity key digest"); + return NULL; + } + if (resolve_my_address(LOG_WARN, options, &addr, NULL, &hostname)<0) { + log_warn(LD_NET, "Couldn't resolve my hostname"); + return NULL; + } + if (!hostname || !strchr(hostname, '.')) { + tor_free(hostname); + hostname = tor_dup_ip(addr); + } + + if (options->VersioningAuthoritativeDir) { + client_versions = + format_recommended_version_list(options->RecommendedClientVersions, 0); + server_versions = + format_recommended_version_list(options->RecommendedServerVersions, 0); + } + + contact = get_options()->ContactInfo; + if (!contact) + contact = "(none)"; + + /* + * Do this so dirserv_compute_performance_thresholds() and + * set_routerstatus_from_routerinfo() see up-to-date bandwidth info. + */ + if (options->V3BandwidthsFile) { + dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL); + } else { + /* + * No bandwidths file; clear the measured bandwidth cache in case we had + * one last time around. + */ + if (dirserv_get_measured_bw_cache_size() > 0) { + dirserv_clear_measured_bw_cache(); + } + } + + /* precompute this part, since we need it to decide what "stable" + * means. */ + SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ri, { + dirserv_set_router_is_running(ri, now); + }); + + routers = smartlist_new(); + smartlist_add_all(routers, rl->routers); + routers_make_ed_keys_unique(routers); + /* After this point, don't use rl->routers; use 'routers' instead. */ + routers_sort_by_identity(routers); + omit_as_sybil = get_possible_sybil_list(routers); + + DIGESTMAP_FOREACH(omit_as_sybil, sybil_id, void *, ignore) { + (void) ignore; + rep_hist_make_router_pessimal(sybil_id, now); + } DIGESTMAP_FOREACH_END; + + /* Count how many have measured bandwidths so we know how to assign flags; + * this must come before dirserv_compute_performance_thresholds() */ + dirserv_count_measured_bws(routers); + + dirserv_compute_performance_thresholds(omit_as_sybil); + + routerstatuses = smartlist_new(); + microdescriptors = smartlist_new(); + + SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { + /* If it has a protover list and contains a protocol name greater than + * MAX_PROTOCOL_NAME_LENGTH, skip it. */ + if (ri->protocol_list && + protover_contains_long_protocol_names(ri->protocol_list)) { + continue; + } + if (ri->cache_info.published_on >= cutoff) { + routerstatus_t *rs; + vote_routerstatus_t *vrs; + node_t *node = node_get_mutable_by_id(ri->cache_info.identity_digest); + if (!node) + continue; + + vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); + rs = &vrs->status; + set_routerstatus_from_routerinfo(rs, node, ri, now, + listbadexits); + + if (ri->cache_info.signing_key_cert) { + memcpy(vrs->ed25519_id, + ri->cache_info.signing_key_cert->signing_key.pubkey, + ED25519_PUBKEY_LEN); + } + + if (digestmap_get(omit_as_sybil, ri->cache_info.identity_digest)) + clear_status_flags_on_sybil(rs); + + if (!vote_on_reachability) + rs->is_flagged_running = 0; + + vrs->version = version_from_platform(ri->platform); + if (ri->protocol_list) { + vrs->protocols = tor_strdup(ri->protocol_list); + } else { + vrs->protocols = tor_strdup( + protover_compute_for_old_tor(vrs->version)); + } + vrs->microdesc = dirvote_format_all_microdesc_vote_lines(ri, now, + microdescriptors); + + smartlist_add(routerstatuses, vrs); + } + } SMARTLIST_FOREACH_END(ri); + + { + smartlist_t *added = + microdescs_add_list_to_cache(get_microdesc_cache(), + microdescriptors, SAVED_NOWHERE, 0); + smartlist_free(added); + smartlist_free(microdescriptors); + } + + smartlist_free(routers); + digestmap_free(omit_as_sybil, NULL); + + /* Apply guardfraction information to routerstatuses. */ + if (options->GuardfractionFile) { + dirserv_read_guardfraction_file(options->GuardfractionFile, + routerstatuses); + } + + /* This pass through applies the measured bw lines to the routerstatuses */ + if (options->V3BandwidthsFile) { + dirserv_read_measured_bandwidths(options->V3BandwidthsFile, + routerstatuses); + } else { + /* + * No bandwidths file; clear the measured bandwidth cache in case we had + * one last time around. + */ + if (dirserv_get_measured_bw_cache_size() > 0) { + dirserv_clear_measured_bw_cache(); + } + } + + v3_out = tor_malloc_zero(sizeof(networkstatus_t)); + + v3_out->type = NS_TYPE_VOTE; + dirvote_get_preferred_voting_intervals(&timing); + v3_out->published = now; + { + char tbuf[ISO_TIME_LEN+1]; + networkstatus_t *current_consensus = + networkstatus_get_live_consensus(now); + long last_consensus_interval; /* only used to pick a valid_after */ + if (current_consensus) + last_consensus_interval = current_consensus->fresh_until - + current_consensus->valid_after; + else + last_consensus_interval = options->TestingV3AuthInitialVotingInterval; + v3_out->valid_after = + voting_schedule_get_start_of_next_interval(now, + (int)last_consensus_interval, + options->TestingV3AuthVotingStartOffset); + format_iso_time(tbuf, v3_out->valid_after); + log_notice(LD_DIR,"Choosing valid-after time in vote as %s: " + "consensus_set=%d, last_interval=%d", + tbuf, current_consensus?1:0, (int)last_consensus_interval); + } + v3_out->fresh_until = v3_out->valid_after + timing.vote_interval; + v3_out->valid_until = v3_out->valid_after + + (timing.vote_interval * timing.n_intervals_valid); + v3_out->vote_seconds = timing.vote_delay; + v3_out->dist_seconds = timing.dist_delay; + tor_assert(v3_out->vote_seconds > 0); + tor_assert(v3_out->dist_seconds > 0); + tor_assert(timing.n_intervals_valid > 0); + + v3_out->client_versions = client_versions; + v3_out->server_versions = server_versions; + + /* These are hardwired, to avoid disaster. */ + v3_out->recommended_relay_protocols = + tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 " + "Link=4 Microdesc=1-2 Relay=2"); + v3_out->recommended_client_protocols = + tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 " + "Link=4 Microdesc=1-2 Relay=2"); + v3_out->required_client_protocols = + tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 " + "Link=4 Microdesc=1-2 Relay=2"); + v3_out->required_relay_protocols = + tor_strdup("Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 " + "Link=3-4 Microdesc=1 Relay=1-2"); + + /* We are not allowed to vote to require anything we don't have. */ + tor_assert(protover_all_supported(v3_out->required_relay_protocols, NULL)); + tor_assert(protover_all_supported(v3_out->required_client_protocols, NULL)); + + /* We should not recommend anything we don't have. */ + tor_assert_nonfatal(protover_all_supported( + v3_out->recommended_relay_protocols, NULL)); + tor_assert_nonfatal(protover_all_supported( + v3_out->recommended_client_protocols, NULL)); + + v3_out->package_lines = smartlist_new(); + { + config_line_t *cl; + for (cl = get_options()->RecommendedPackages; cl; cl = cl->next) { + if (validate_recommended_package_line(cl->value)) + smartlist_add_strdup(v3_out->package_lines, cl->value); + } + } + + v3_out->known_flags = smartlist_new(); + smartlist_split_string(v3_out->known_flags, + "Authority Exit Fast Guard Stable V2Dir Valid HSDir", + 0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); + if (vote_on_reachability) + smartlist_add_strdup(v3_out->known_flags, "Running"); + if (listbadexits) + smartlist_add_strdup(v3_out->known_flags, "BadExit"); + smartlist_sort_strings(v3_out->known_flags); + + if (options->ConsensusParams) { + v3_out->net_params = smartlist_new(); + smartlist_split_string(v3_out->net_params, + options->ConsensusParams, NULL, 0, 0); + smartlist_sort_strings(v3_out->net_params); + } + + voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); + voter->nickname = tor_strdup(options->Nickname); + memcpy(voter->identity_digest, identity_digest, DIGEST_LEN); + voter->sigs = smartlist_new(); + voter->address = hostname; + voter->addr = addr; + voter->dir_port = router_get_advertised_dir_port(options, 0); + voter->or_port = router_get_advertised_or_port(options); + voter->contact = tor_strdup(contact); + if (options->V3AuthUseLegacyKey) { + authority_cert_t *c = get_my_v3_legacy_cert(); + if (c) { + if (crypto_pk_get_digest(c->identity_key, voter->legacy_id_digest)) { + log_warn(LD_BUG, "Unable to compute digest of legacy v3 identity key"); + memset(voter->legacy_id_digest, 0, DIGEST_LEN); + } + } + } + + v3_out->voters = smartlist_new(); + smartlist_add(v3_out->voters, voter); + v3_out->cert = authority_cert_dup(cert); + v3_out->routerstatus_list = routerstatuses; + /* Note: networkstatus_digest is unset; it won't get set until we actually + * format the vote. */ + + return v3_out; } diff --git a/src/or/dirvote.h b/src/or/dirauth/dirvote.h index deeb27bfe1..b69bbbf5d9 100644 --- a/src/or/dirvote.h +++ b/src/or/dirauth/dirvote.h @@ -12,8 +12,6 @@ #ifndef TOR_DIRVOTE_H #define TOR_DIRVOTE_H -#include "testsupport.h" - /* * Ideally, assuming synced clocks, we should only need 1 second for each of: * - Vote @@ -56,57 +54,11 @@ #define ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD 0 /** The lowest consensus method that we currently support. */ -#define MIN_SUPPORTED_CONSENSUS_METHOD 13 +#define MIN_SUPPORTED_CONSENSUS_METHOD 25 /** The highest consensus method that we currently support. */ #define MAX_SUPPORTED_CONSENSUS_METHOD 28 -/** Lowest consensus method where microdesc consensuses omit any entry - * with no microdesc. */ -#define MIN_METHOD_FOR_MANDATORY_MICRODESC 13 - -/** Lowest consensus method that contains "a" lines. */ -#define MIN_METHOD_FOR_A_LINES 14 - -/** Lowest consensus method where microdescs may include a "p6" line. */ -#define MIN_METHOD_FOR_P6_LINES 15 - -/** Lowest consensus method where microdescs may include an onion-key-ntor - * line */ -#define MIN_METHOD_FOR_NTOR_KEY 16 - -/** Lowest consensus method that ensures that authorities output an - * Unmeasured=1 flag for unmeasured bandwidths */ -#define MIN_METHOD_TO_CLIP_UNMEASURED_BW 17 - -/** Lowest consensus method where authorities may include an "id" line in - * microdescriptors. */ -#define MIN_METHOD_FOR_ID_HASH_IN_MD 18 - -/** Lowest consensus method where we include "package" lines*/ -#define MIN_METHOD_FOR_PACKAGE_LINES 19 - -/** Lowest consensus method where authorities may include - * GuardFraction information in microdescriptors. */ -#define MIN_METHOD_FOR_GUARDFRACTION 20 - -/** Lowest consensus method where authorities may include an "id" line for - * ed25519 identities in microdescriptors. (Broken; see - * consensus_method_is_supported() for more info.) */ -#define MIN_METHOD_FOR_ED25519_ID_IN_MD 21 - -/** Lowest consensus method where authorities vote on ed25519 ids and ensure - * ed25519 id consistency. */ -#define MIN_METHOD_FOR_ED25519_ID_VOTING 22 - -/** Lowest consensus method where authorities may include a shared random - * value(s). */ -#define MIN_METHOD_FOR_SHARED_RANDOM 23 - -/** Lowest consensus method where authorities drop all nodes that don't get - * the Valid flag. */ -#define MIN_METHOD_FOR_EXCLUDING_INVALID_NODES 24 - /** Lowest consensus method where authorities vote on required/recommended * protocols. */ #define MIN_METHOD_FOR_RECOMMENDED_PROTOCOLS 25 @@ -132,74 +84,27 @@ * get confused with the above macros.) */ #define DEFAULT_MAX_UNMEASURED_BW_KB 20 +/* Directory Get Vote (DGV) flags for dirvote_get_vote(). */ +#define DGV_BY_ID 1 +#define DGV_INCLUDE_PENDING 2 +#define DGV_INCLUDE_PREVIOUS 4 + +/* + * Public API. Used outside of the dirauth subsystem. + * + * We need to nullify them if the module is disabled. + */ +#ifdef HAVE_MODULE_DIRAUTH + +time_t dirvote_act(const or_options_t *options, time_t now); void dirvote_free_all(void); -/* vote manipulation */ -char *networkstatus_compute_consensus(smartlist_t *votes, - int total_authorities, - crypto_pk_t *identity_key, - crypto_pk_t *signing_key, - const char *legacy_identity_key_digest, - crypto_pk_t *legacy_signing_key, - consensus_flavor_t flavor); -int networkstatus_add_detached_signatures(networkstatus_t *target, - ns_detached_signatures_t *sigs, - const char *source, - int severity, - const char **msg_out); -char *networkstatus_get_detached_signatures(smartlist_t *consensuses); -void ns_detached_signatures_free_(ns_detached_signatures_t *s); -#define ns_detached_signatures_free(s) \ - FREE_AND_NULL(ns_detached_signatures_t, ns_detached_signatures_free_, (s)) - -/* cert manipulation */ -authority_cert_t *authority_cert_dup(authority_cert_t *cert); - -/* vote scheduling */ - -/** Scheduling information for a voting interval. */ -typedef struct { - /** When do we generate and distribute our vote for this interval? */ - time_t voting_starts; - /** When do we send an HTTP request for any votes that we haven't - * been posted yet?*/ - time_t fetch_missing_votes; - /** When do we give up on getting more votes and generate a consensus? */ - time_t voting_ends; - /** When do we send an HTTP request for any signatures we're expecting to - * see on the consensus? */ - time_t fetch_missing_signatures; - /** When do we publish the consensus? */ - time_t interval_starts; - - /* True iff we have generated and distributed our vote. */ - int have_voted; - /* True iff we've requested missing votes. */ - int have_fetched_missing_votes; - /* True iff we have built a consensus and sent the signatures around. */ - int have_built_consensus; - /* True iff we've fetched missing signatures. */ - int have_fetched_missing_signatures; - /* True iff we have published our consensus. */ - int have_published_consensus; - - /* True iff this voting schedule was set on demand meaning not through the - * normal vote operation of a dirauth or when a consensus is set. This only - * applies to a directory authority that needs to recalculate the voting - * timings only for the first vote even though this object was initilized - * prior to voting. */ - int created_on_demand; -} voting_schedule_t; - -void dirvote_get_preferred_voting_intervals(vote_timing_t *timing_out); -time_t dirvote_get_start_of_next_interval(time_t now, - int interval, - int offset); -void dirvote_recalculate_timing(const or_options_t *options, time_t now); -void dirvote_act(const or_options_t *options, time_t now); -time_t dirvote_get_next_valid_after_time(void); - -/* invoked on timers and by outside triggers. */ +void dirvote_parse_sr_commits(networkstatus_t *ns, const smartlist_t *tokens); +void dirvote_clear_commits(networkstatus_t *ns); +void dirvote_dirreq_get_status_vote(const char *url, smartlist_t *items, + smartlist_t *dir_items); + +/* Storing signatures and votes functions */ struct pending_vote_t * dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out); @@ -207,15 +112,82 @@ int dirvote_add_signatures(const char *detached_signatures_body, const char *source, const char **msg_out); +#else /* HAVE_MODULE_DIRAUTH */ + +static inline time_t +dirvote_act(const or_options_t *options, time_t now) +{ + (void) options; + (void) now; + return TIME_MAX; +} + +static inline void +dirvote_free_all(void) +{ +} + +static inline void +dirvote_parse_sr_commits(networkstatus_t *ns, const smartlist_t *tokens) +{ + (void) ns; + (void) tokens; +} + +static inline void +dirvote_clear_commits(networkstatus_t *ns) +{ + (void) ns; +} + +static inline void +dirvote_dirreq_get_status_vote(const char *url, smartlist_t *items, + smartlist_t *dir_items) +{ + (void) url; + (void) items; + (void) dir_items; +} + +static inline struct pending_vote_t * +dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out) +{ + (void) vote_body; + /* If the dirauth module is disabled, this should NEVER be called else we + * failed to safeguard the dirauth module. */ + tor_assert_nonfatal_unreached(); + + /* We need to send out an error code. */ + *status_out = 400; + *msg_out = "No directory authority support"; + return NULL; +} + +static inline int +dirvote_add_signatures(const char *detached_signatures_body, const char *source, + const char **msg_out) +{ + (void) detached_signatures_body; + (void) source; + (void) msg_out; + /* If the dirauth module is disabled, this should NEVER be called else we + * failed to safeguard the dirauth module. */ + tor_assert_nonfatal_unreached(); + return 0; +} + +#endif /* HAVE_MODULE_DIRAUTH */ + /* Item access */ MOCK_DECL(const char*, dirvote_get_pending_consensus, (consensus_flavor_t flav)); MOCK_DECL(const char*, dirvote_get_pending_detached_signatures, (void)); - -#define DGV_BY_ID 1 -#define DGV_INCLUDE_PENDING 2 -#define DGV_INCLUDE_PREVIOUS 4 const cached_dir_t *dirvote_get_vote(const char *fp, int flags); + +/* + * API used _only_ by the dirauth subsystem. + */ + void set_routerstatus_from_routerinfo(routerstatus_t *rs, node_t *node, routerinfo_t *ri, time_t now, @@ -224,26 +196,18 @@ networkstatus_t * dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, authority_cert_t *cert); -microdesc_t *dirvote_create_microdescriptor(const routerinfo_t *ri, - int consensus_method); -ssize_t dirvote_format_microdesc_vote_line(char *out, size_t out_len, - const microdesc_t *md, - int consensus_method_low, - int consensus_method_high); vote_microdesc_hash_t *dirvote_format_all_microdesc_vote_lines( const routerinfo_t *ri, time_t now, smartlist_t *microdescriptors_out); -int vote_routerstatus_find_microdesc_hash(char *digest256_out, - const vote_routerstatus_t *vrs, - int method, - digest_algorithm_t alg); -document_signature_t *voter_get_sig_by_algorithm( - const networkstatus_voter_info_t *voter, - digest_algorithm_t alg); - +/* + * Exposed functions for unit tests. + */ #ifdef DIRVOTE_PRIVATE + +/* Cert manipulation */ +STATIC authority_cert_t *authority_cert_dup(authority_cert_t *cert); STATIC int32_t dirvote_get_intermediate_param_value( const smartlist_t *param_list, const char *keyword, @@ -258,6 +222,25 @@ STATIC int networkstatus_compute_bw_weights_v10(smartlist_t *chunks, int64_t G, int64_t M, int64_t E, int64_t D, int64_t T, int64_t weight_scale); +STATIC +char *networkstatus_compute_consensus(smartlist_t *votes, + int total_authorities, + crypto_pk_t *identity_key, + crypto_pk_t *signing_key, + const char *legacy_identity_key_digest, + crypto_pk_t *legacy_signing_key, + consensus_flavor_t flavor); +STATIC +int networkstatus_add_detached_signatures(networkstatus_t *target, + ns_detached_signatures_t *sigs, + const char *source, + int severity, + const char **msg_out); +STATIC +char *networkstatus_get_detached_signatures(smartlist_t *consensuses); +STATIC microdesc_t *dirvote_create_microdescriptor(const routerinfo_t *ri, + int consensus_method); + #endif /* defined(DIRVOTE_PRIVATE) */ #endif /* !defined(TOR_DIRVOTE_H) */ diff --git a/src/or/dirauth/mode.h b/src/or/dirauth/mode.h new file mode 100644 index 0000000000..8a0d3142f1 --- /dev/null +++ b/src/or/dirauth/mode.h @@ -0,0 +1,38 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file mode.h + * \brief Standalone header file for directory authority mode. + **/ + +#ifndef TOR_DIRAUTH_MODE_H +#define TOR_DIRAUTH_MODE_H + +#ifdef HAVE_MODULE_DIRAUTH + +#include "router.h" + +/* Return true iff we believe ourselves to be a v3 authoritative directory + * server. */ +static inline int +authdir_mode_v3(const or_options_t *options) +{ + return authdir_mode(options) && options->V3AuthoritativeDir != 0; +} + +#else /* HAVE_MODULE_DIRAUTH */ + +/* Without the dirauth module, we can't be a v3 directory authority, ever. */ + +static inline int +authdir_mode_v3(const or_options_t *options) +{ + (void) options; + return 0; +} + +#endif /* HAVE_MODULE_DIRAUTH */ + +#endif /* TOR_MODE_H */ + diff --git a/src/or/shared_random.c b/src/or/dirauth/shared_random.c index 13416d6bc7..6dd1f330e0 100644 --- a/src/or/shared_random.c +++ b/src/or/dirauth/shared_random.c @@ -91,13 +91,19 @@ #include "shared_random.h" #include "config.h" #include "confparse.h" -#include "dirvote.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "networkstatus.h" -#include "routerkeys.h" #include "router.h" +#include "routerkeys.h" #include "routerlist.h" +#include "shared_random_client.h" #include "shared_random_state.h" #include "util.h" +#include "voting_schedule.h" + +#include "dirauth/dirvote.h" +#include "dirauth/mode.h" /* String prefix of shared random values in votes/consensuses. */ static const char previous_srv_str[] = "shared-rand-previous-value"; @@ -498,20 +504,6 @@ get_vote_line_from_commit(const sr_commit_t *commit, sr_phase_t phase) return vote_line; } -/* Convert a given srv object to a string for the control port. This doesn't - * fail and the srv object MUST be valid. */ -static char * -srv_to_control_string(const sr_srv_t *srv) -{ - char *srv_str; - char srv_hash_encoded[SR_SRV_VALUE_BASE64_LEN + 1]; - tor_assert(srv); - - sr_srv_encode(srv_hash_encoded, sizeof(srv_hash_encoded), srv); - tor_asprintf(&srv_str, "%s", srv_hash_encoded); - return srv_str; -} - /* Return a heap allocated string that contains the given <b>srv</b> string * representation formatted for a networkstatus document using the * <b>key</b> as the start of the line. This doesn't return NULL. */ @@ -874,27 +866,6 @@ get_majority_srv_from_votes(const smartlist_t *votes, int current) return the_srv; } -/* Encode the given shared random value and put it in dst. Destination - * buffer must be at least SR_SRV_VALUE_BASE64_LEN plus the NULL byte. */ -void -sr_srv_encode(char *dst, size_t dst_len, const sr_srv_t *srv) -{ - int ret; - /* Extra byte for the NULL terminated char. */ - char buf[SR_SRV_VALUE_BASE64_LEN + 1]; - - tor_assert(dst); - tor_assert(srv); - tor_assert(dst_len >= sizeof(buf)); - - ret = base64_encode(buf, sizeof(buf), (const char *) srv->value, - sizeof(srv->value), 0); - /* Always expect the full length without the NULL byte. */ - tor_assert(ret == (sizeof(buf) - 1)); - tor_assert(ret <= (int) dst_len); - strlcpy(dst, buf, dst_len); -} - /* Free a commit object. */ void sr_commit_free_(sr_commit_t *commit) @@ -1036,55 +1007,6 @@ sr_compute_srv(void) tor_free(reveals); } -/* Parse a list of arguments from a SRV value either from a vote, consensus - * or from our disk state and return a newly allocated srv object. NULL is - * returned on error. - * - * The arguments' order: - * num_reveals, value - */ -sr_srv_t * -sr_parse_srv(const smartlist_t *args) -{ - char *value; - int ok, ret; - uint64_t num_reveals; - sr_srv_t *srv = NULL; - - tor_assert(args); - - if (smartlist_len(args) < 2) { - goto end; - } - - /* First argument is the number of reveal values */ - num_reveals = tor_parse_uint64(smartlist_get(args, 0), - 10, 0, UINT64_MAX, &ok, NULL); - if (!ok) { - goto end; - } - /* Second and last argument is the shared random value it self. */ - value = smartlist_get(args, 1); - if (strlen(value) != SR_SRV_VALUE_BASE64_LEN) { - goto end; - } - - srv = tor_malloc_zero(sizeof(*srv)); - srv->num_reveals = num_reveals; - /* We subtract one byte from the srclen because the function ignores the - * '=' character in the given buffer. This is broken but it's a documented - * behavior of the implementation. */ - ret = base64_decode((char *) srv->value, sizeof(srv->value), value, - SR_SRV_VALUE_BASE64_LEN - 1); - if (ret != sizeof(srv->value)) { - tor_free(srv); - srv = NULL; - goto end; - } - end: - return srv; -} - /* Parse a commit from a vote or from our disk state and return a newly * allocated commit object. NULL is returned on error. * @@ -1333,7 +1255,7 @@ sr_act_post_consensus(const networkstatus_t *consensus) } /* Prepare our state so that it's ready for the next voting period. */ - sr_state_update(dirvote_get_next_valid_after_time()); + sr_state_update(voting_schedule_get_next_valid_after_time()); } /* Initialize shared random subsystem. This MUST be called early in the boot @@ -1352,84 +1274,6 @@ sr_save_and_cleanup(void) sr_cleanup(); } -/* Return the current SRV string representation for the control port. Return a - * newly allocated string on success containing the value else "" if not found - * or if we don't have a valid consensus yet. */ -char * -sr_get_current_for_control(void) -{ - char *srv_str; - const networkstatus_t *c = networkstatus_get_latest_consensus(); - if (c && c->sr_info.current_srv) { - srv_str = srv_to_control_string(c->sr_info.current_srv); - } else { - srv_str = tor_strdup(""); - } - return srv_str; -} - -/* Return the previous SRV string representation for the control port. Return - * a newly allocated string on success containing the value else "" if not - * found or if we don't have a valid consensus yet. */ -char * -sr_get_previous_for_control(void) -{ - char *srv_str; - const networkstatus_t *c = networkstatus_get_latest_consensus(); - if (c && c->sr_info.previous_srv) { - srv_str = srv_to_control_string(c->sr_info.previous_srv); - } else { - srv_str = tor_strdup(""); - } - return srv_str; -} - -/* Return current shared random value from the latest consensus. Caller can - * NOT keep a reference to the returned pointer. Return NULL if none. */ -const sr_srv_t * -sr_get_current(const networkstatus_t *ns) -{ - const networkstatus_t *consensus; - - /* Use provided ns else get a live one */ - if (ns) { - consensus = ns; - } else { - consensus = networkstatus_get_live_consensus(approx_time()); - } - /* Ideally we would never be asked for an SRV without a live consensus. Make - * sure this assumption is correct. */ - tor_assert_nonfatal(consensus); - - if (consensus) { - return consensus->sr_info.current_srv; - } - return NULL; -} - -/* Return previous shared random value from the latest consensus. Caller can - * NOT keep a reference to the returned pointer. Return NULL if none. */ -const sr_srv_t * -sr_get_previous(const networkstatus_t *ns) -{ - const networkstatus_t *consensus; - - /* Use provided ns else get a live one */ - if (ns) { - consensus = ns; - } else { - consensus = networkstatus_get_live_consensus(approx_time()); - } - /* Ideally we would never be asked for an SRV without a live consensus. Make - * sure this assumption is correct. */ - tor_assert_nonfatal(consensus); - - if (consensus) { - return consensus->sr_info.previous_srv; - } - return NULL; -} - #ifdef TOR_UNIT_TESTS /* Set the global value of number of SRV agreements so the test can play diff --git a/src/or/shared_random.h b/src/or/dirauth/shared_random.h index 675a8d8b06..1778ce8f09 100644 --- a/src/or/shared_random.h +++ b/src/or/dirauth/shared_random.h @@ -101,21 +101,48 @@ typedef struct sr_commit_t { /* API */ -/* Public methods: */ +/* Public methods used _outside_ of the module. + * + * We need to nullify them if the module is disabled. */ +#ifdef HAVE_MODULE_DIRAUTH int sr_init(int save_to_disk); void sr_save_and_cleanup(void); void sr_act_post_consensus(const networkstatus_t *consensus); + +#else /* HAVE_MODULE_DIRAUTH */ + +static inline int +sr_init(int save_to_disk) +{ + (void) save_to_disk; + /* Always return success. */ + return 0; +} + +static inline void +sr_save_and_cleanup(void) +{ +} + +static inline void +sr_act_post_consensus(const networkstatus_t *consensus) +{ + (void) consensus; +} + +#endif /* HAVE_MODULE_DIRAUTH */ + +/* Public methods used only by dirauth code. */ + void sr_handle_received_commits(smartlist_t *commits, crypto_pk_t *voter_key); sr_commit_t *sr_parse_commit(const smartlist_t *args); -sr_srv_t *sr_parse_srv(const smartlist_t *args); char *sr_get_string_for_vote(void); char *sr_get_string_for_consensus(const smartlist_t *votes, int32_t num_srv_agreements); void sr_commit_free_(sr_commit_t *commit); #define sr_commit_free(sr) FREE_AND_NULL(sr_commit_t, sr_commit_free_, (sr)) -void sr_srv_encode(char *dst, size_t dst_len, const sr_srv_t *srv); /* Private methods (only used by shared_random_state.c): */ static inline @@ -128,12 +155,6 @@ void sr_compute_srv(void); sr_commit_t *sr_generate_our_commit(time_t timestamp, const authority_cert_t *my_rsa_cert); -char *sr_get_current_for_control(void); -char *sr_get_previous_for_control(void); - -const sr_srv_t *sr_get_current(const networkstatus_t *ns); -const sr_srv_t *sr_get_previous(const networkstatus_t *ns); - #ifdef SHARED_RANDOM_PRIVATE /* Encode */ diff --git a/src/or/shared_random_state.c b/src/or/dirauth/shared_random_state.c index 7bac8e9482..fc0e4e5630 100644 --- a/src/or/shared_random_state.c +++ b/src/or/dirauth/shared_random_state.c @@ -11,13 +11,16 @@ #define SHARED_RANDOM_STATE_PRIVATE #include "or.h" -#include "shared_random.h" #include "config.h" #include "confparse.h" -#include "dirvote.h" +#include "crypto_util.h" +#include "dirauth/dirvote.h" #include "networkstatus.h" #include "router.h" +#include "shared_random.h" +#include "shared_random_client.h" #include "shared_random_state.h" +#include "voting_schedule.h" /* Default filename of the shared random state on disk. */ static const char default_fname[] = "sr-state"; @@ -53,10 +56,6 @@ DUMMY_TYPECHECK_INSTANCE(sr_disk_state_t); VAR(#member, conftype, member, initvalue) /* Our persistent state magic number. */ #define SR_DISK_STATE_MAGIC 0x98AB1254 -/* Each protocol phase has 12 rounds */ -#define SHARED_RANDOM_N_ROUNDS 12 -/* Number of phase we have in a protocol. */ -#define SHARED_RANDOM_N_PHASES 2 static int disk_state_validate_cb(void *old_state, void *state, void *default_state, @@ -115,81 +114,6 @@ get_phase_str(sr_phase_t phase) return the_string; } - -/* Return the voting interval of the tor vote subsystem. */ -static int -get_voting_interval(void) -{ - int interval; - networkstatus_t *consensus = networkstatus_get_live_consensus(time(NULL)); - - if (consensus) { - interval = (int)(consensus->fresh_until - consensus->valid_after); - } else { - /* Same for both a testing and real network. We voluntarily ignore the - * InitialVotingInterval since it complexifies things and it doesn't - * affect the SR protocol. */ - interval = get_options()->V3AuthVotingInterval; - } - tor_assert(interval > 0); - return interval; -} - -/* Given the time <b>now</b>, return the start time of the current round of - * the SR protocol. For example, if it's 23:47:08, the current round thus - * started at 23:47:00 for a voting interval of 10 seconds. */ -STATIC time_t -get_start_time_of_current_round(void) -{ - const or_options_t *options = get_options(); - int voting_interval = get_voting_interval(); - /* First, get the start time of the next round */ - time_t next_start = dirvote_get_next_valid_after_time(); - /* Now roll back next_start by a voting interval to find the start time of - the current round. */ - time_t curr_start = dirvote_get_start_of_next_interval( - next_start - voting_interval - 1, - voting_interval, - options->TestingV3AuthVotingStartOffset); - return curr_start; -} - -/** Return the start time of the current SR protocol run. For example, if the - * time is 23/06/2017 23:47:08 and a full SR protocol run is 24 hours, this - * function should return 23/06/2017 00:00:00. */ -time_t -sr_state_get_start_time_of_current_protocol_run(time_t now) -{ - int total_rounds = SHARED_RANDOM_N_ROUNDS * SHARED_RANDOM_N_PHASES; - int voting_interval = get_voting_interval(); - /* Find the time the current round started. */ - time_t beginning_of_current_round = get_start_time_of_current_round(); - - /* Get current SR protocol round */ - int current_round = (now / voting_interval) % total_rounds; - - /* Get start time by subtracting the time elapsed from the beginning of the - protocol run */ - time_t time_elapsed_since_start_of_run = current_round * voting_interval; - return beginning_of_current_round - time_elapsed_since_start_of_run; -} - -/** Return the time (in seconds) it takes to complete a full SR protocol phase - * (e.g. the commit phase). */ -unsigned int -sr_state_get_phase_duration(void) -{ - return SHARED_RANDOM_N_ROUNDS * get_voting_interval(); -} - -/** Return the time (in seconds) it takes to complete a full SR protocol run */ -unsigned int -sr_state_get_protocol_run_duration(void) -{ - int total_protocol_rounds = SHARED_RANDOM_N_ROUNDS * SHARED_RANDOM_N_PHASES; - return total_protocol_rounds * get_voting_interval(); -} - /* Return the time we should expire the state file created at <b>now</b>. * We expire the state file in the beginning of the next protocol run. */ STATIC time_t @@ -1370,7 +1294,7 @@ sr_state_init(int save_to_disk, int read_from_disk) /* We have a state in memory, let's make sure it's updated for the current * and next voting round. */ { - time_t valid_after = dirvote_get_next_valid_after_time(); + time_t valid_after = voting_schedule_get_next_valid_after_time(); sr_state_update(valid_after); } return 0; diff --git a/src/or/shared_random_state.h b/src/or/dirauth/shared_random_state.h index fdbbf4919a..60a326f86c 100644 --- a/src/or/shared_random_state.h +++ b/src/or/dirauth/shared_random_state.h @@ -121,16 +121,11 @@ int sr_state_is_initialized(void); void sr_state_save(void); void sr_state_free_all(void); -time_t sr_state_get_start_time_of_current_protocol_run(time_t now); -unsigned int sr_state_get_phase_duration(void); -unsigned int sr_state_get_protocol_run_duration(void); - #ifdef SHARED_RANDOM_STATE_PRIVATE STATIC int disk_state_load_from_disk_impl(const char *fname); STATIC sr_phase_t get_sr_protocol_phase(time_t valid_after); -STATIC time_t get_start_time_of_current_round(void); STATIC time_t get_state_valid_until_time(time_t now); STATIC const char *get_phase_str(sr_phase_t phase); diff --git a/src/or/directory.c b/src/or/directory.c index 8636f68410..ca53756cbe 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -18,9 +18,10 @@ #include "consdiffmgr.h" #include "control.h" #include "compat.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "directory.h" #include "dirserv.h" -#include "dirvote.h" #include "entrynodes.h" #include "geoip.h" #include "hs_cache.h" @@ -41,7 +42,6 @@ #include "routerlist.h" #include "routerparse.h" #include "routerset.h" -#include "shared_random.h" #if defined(EXPORTMALLINFO) && defined(HAVE_MALLOC_H) && defined(HAVE_MALLINFO) #if !defined(OpenBSD) @@ -49,6 +49,10 @@ #endif #endif +#include "dirauth/dirvote.h" +#include "dirauth/mode.h" +#include "dirauth/shared_random.h" + /** * \file directory.c * \brief Code to send and fetch information from directory authorities and @@ -794,9 +798,9 @@ directory_choose_address_routerstatus(const routerstatus_t *status, * Use the preferred address and port if they are reachable, otherwise, * use the alternate address and port (if any). */ - have_or = fascist_firewall_choose_address_rs(status, - FIREWALL_OR_CONNECTION, 0, - use_or_ap); + fascist_firewall_choose_address_rs(status, FIREWALL_OR_CONNECTION, 0, + use_or_ap); + have_or = tor_addr_port_is_valid_ap(use_or_ap, 0); } /* DirPort connections @@ -805,9 +809,9 @@ directory_choose_address_routerstatus(const routerstatus_t *status, indirection == DIRIND_ANON_DIRPORT || (indirection == DIRIND_ONEHOP && !directory_must_use_begindir(options))) { - have_dir = fascist_firewall_choose_address_rs(status, - FIREWALL_DIR_CONNECTION, 0, - use_dir_ap); + fascist_firewall_choose_address_rs(status, FIREWALL_DIR_CONNECTION, 0, + use_dir_ap); + have_dir = tor_addr_port_is_valid_ap(use_dir_ap, 0); } /* We rejected all addresses in the relay's status. This means we can't @@ -2438,7 +2442,7 @@ connection_dir_client_reached_eof(dir_connection_t *conn) * and the date header. (We used to check now-date_header, but that's * inaccurate if we spend a lot of time downloading.) */ - apparent_skew = conn->base_.timestamp_lastwritten - date_header; + apparent_skew = conn->base_.timestamp_last_write_allowed - date_header; if (labs(apparent_skew)>ALLOW_DIRECTORY_TIME_SKEW) { int trusted = router_digest_is_trusted_dir(conn->identity_digest); clock_skew_warning(TO_CONN(conn), apparent_skew, trusted, LD_HTTP, @@ -4439,59 +4443,15 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args) { const char *url = args->url; { - int current; ssize_t body_len = 0; ssize_t estimated_len = 0; + int lifetime = 60; /* XXXX?? should actually use vote intervals. */ /* This smartlist holds strings that we can compress on the fly. */ smartlist_t *items = smartlist_new(); /* This smartlist holds cached_dir_t objects that have a precompressed * deflated version. */ smartlist_t *dir_items = smartlist_new(); - int lifetime = 60; /* XXXX?? should actually use vote intervals. */ - url += strlen("/tor/status-vote/"); - current = !strcmpstart(url, "current/"); - url = strchr(url, '/'); - tor_assert(url); - ++url; - if (!strcmp(url, "consensus")) { - const char *item; - tor_assert(!current); /* we handle current consensus specially above, - * since it wants to be spooled. */ - if ((item = dirvote_get_pending_consensus(FLAV_NS))) - smartlist_add(items, (char*)item); - } else if (!current && !strcmp(url, "consensus-signatures")) { - /* XXXX the spec says that we should implement - * current/consensus-signatures too. It doesn't seem to be needed, - * though. */ - const char *item; - if ((item=dirvote_get_pending_detached_signatures())) - smartlist_add(items, (char*)item); - } else if (!strcmp(url, "authority")) { - const cached_dir_t *d; - int flags = DGV_BY_ID | - (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING); - if ((d=dirvote_get_vote(NULL, flags))) - smartlist_add(dir_items, (cached_dir_t*)d); - } else { - const cached_dir_t *d; - smartlist_t *fps = smartlist_new(); - int flags; - if (!strcmpstart(url, "d/")) { - url += 2; - flags = DGV_INCLUDE_PENDING | DGV_INCLUDE_PREVIOUS; - } else { - flags = DGV_BY_ID | - (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING); - } - dir_split_resource_into_fingerprints(url, fps, NULL, - DSR_HEX|DSR_SORT_UNIQ); - SMARTLIST_FOREACH(fps, char *, fp, { - if ((d = dirvote_get_vote(fp, flags))) - smartlist_add(dir_items, (cached_dir_t*)d); - tor_free(fp); - }); - smartlist_free(fps); - } + dirvote_dirreq_get_status_vote(url, items, dir_items); if (!smartlist_len(dir_items) && !smartlist_len(items)) { write_short_http_response(conn, 404, "Not found"); goto vote_done; @@ -5219,6 +5179,9 @@ connection_dir_finished_flushing(dir_connection_t *conn) tor_assert(conn); tor_assert(conn->base_.type == CONN_TYPE_DIR); + if (conn->base_.marked_for_close) + return 0; + /* Note that we have finished writing the directory response. For direct * connections this means we're done; for tunneled connections it's only * an intermediate step. */ @@ -5302,84 +5265,71 @@ connection_dir_finished_connecting(dir_connection_t *conn) /** Decide which download schedule we want to use based on descriptor type * in <b>dls</b> and <b>options</b>. - * Then return a list of int pointers defining download delays in seconds. + * + * Then, return the initial delay for that download schedule, in seconds. + * * Helper function for download_status_increment_failure(), * download_status_reset(), and download_status_increment_attempt(). */ -STATIC const smartlist_t * -find_dl_schedule(const download_status_t *dls, const or_options_t *options) +STATIC int +find_dl_min_delay(const download_status_t *dls, const or_options_t *options) { + tor_assert(dls); + tor_assert(options); + switch (dls->schedule) { case DL_SCHED_GENERIC: /* Any other directory document */ if (dir_server_mode(options)) { /* A directory authority or directory mirror */ - return options->TestingServerDownloadSchedule; + return options->TestingServerDownloadInitialDelay; } else { - return options->TestingClientDownloadSchedule; + return options->TestingClientDownloadInitialDelay; } case DL_SCHED_CONSENSUS: if (!networkstatus_consensus_can_use_multiple_directories(options)) { /* A public relay */ - return options->TestingServerConsensusDownloadSchedule; + return options->TestingServerConsensusDownloadInitialDelay; } else { /* A client or bridge */ if (networkstatus_consensus_is_bootstrapping(time(NULL))) { /* During bootstrapping */ if (!networkstatus_consensus_can_use_extra_fallbacks(options)) { /* A bootstrapping client without extra fallback directories */ - return - options->ClientBootstrapConsensusAuthorityOnlyDownloadSchedule; + return options-> + ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay; } else if (dls->want_authority) { /* A bootstrapping client with extra fallback directories, but * connecting to an authority */ return - options->ClientBootstrapConsensusAuthorityDownloadSchedule; + options->ClientBootstrapConsensusAuthorityDownloadInitialDelay; } else { /* A bootstrapping client connecting to extra fallback directories */ return - options->ClientBootstrapConsensusFallbackDownloadSchedule; + options->ClientBootstrapConsensusFallbackDownloadInitialDelay; } } else { /* A client with a reasonably live consensus, with or without * certificates */ - return options->TestingClientConsensusDownloadSchedule; + return options->TestingClientConsensusDownloadInitialDelay; } } case DL_SCHED_BRIDGE: if (options->UseBridges && num_bridges_usable(0) > 0) { /* A bridge client that is sure that one or more of its bridges are * running can afford to wait longer to update bridge descriptors. */ - return options->TestingBridgeDownloadSchedule; + return options->TestingBridgeDownloadInitialDelay; } else { /* A bridge client which might have no running bridges, must try to * get bridge descriptors straight away. */ - return options->TestingBridgeBootstrapDownloadSchedule; + return options->TestingBridgeBootstrapDownloadInitialDelay; } default: tor_assert(0); } /* Impossible, but gcc will fail with -Werror without a `return`. */ - return NULL; -} - -/** Decide which minimum delay step we want to use based on - * descriptor type in <b>dls</b> and <b>options</b>. - * Helper function for download_status_schedule_get_delay(). */ -STATIC int -find_dl_min_delay(download_status_t *dls, const or_options_t *options) -{ - tor_assert(dls); - tor_assert(options); - - /* - * For now, just use the existing schedule config stuff and pick the - * first/last entries off to get min/max delay for backoff purposes - */ - const smartlist_t *schedule = find_dl_schedule(dls, options); - tor_assert(schedule != NULL && smartlist_len(schedule) >= 2); - return *(int *)(smartlist_get(schedule, 0)); + return 0; } /** As next_random_exponential_delay() below, but does not compute a random @@ -5636,10 +5586,9 @@ download_status_increment_attempt(download_status_t *dls, const char *item, static time_t download_status_get_initial_delay_from_now(const download_status_t *dls) { - const smartlist_t *schedule = find_dl_schedule(dls, get_options()); /* We use constant initial delays, even in exponential backoff * schedules. */ - return time(NULL) + *(int *)smartlist_get(schedule, 0); + return time(NULL) + find_dl_min_delay(dls, get_options()); } /** Reset <b>dls</b> so that it will be considered downloadable @@ -5966,4 +5915,3 @@ dir_split_resource_into_spoolable(const char *resource, smartlist_free(fingerprints); return r; } - diff --git a/src/or/directory.h b/src/or/directory.h index aa4d29a5bb..5f5ff7eca6 100644 --- a/src/or/directory.h +++ b/src/or/directory.h @@ -259,9 +259,7 @@ STATIC char* authdir_type_to_string(dirinfo_type_t auth); STATIC const char * dir_conn_purpose_to_string(int purpose); STATIC int should_use_directory_guards(const or_options_t *options); STATIC compression_level_t choose_compression_level(ssize_t n_bytes); -STATIC const smartlist_t *find_dl_schedule(const download_status_t *dls, - const or_options_t *options); -STATIC int find_dl_min_delay(download_status_t *dls, +STATIC int find_dl_min_delay(const download_status_t *dls, const or_options_t *options); STATIC int next_random_exponential_delay(int delay, diff --git a/src/or/dirserv.c b/src/or/dirserv.c index ced40899d0..4e09c1c65b 100644 --- a/src/or/dirserv.c +++ b/src/or/dirserv.c @@ -18,7 +18,6 @@ #include "control.h" #include "directory.h" #include "dirserv.h" -#include "dirvote.h" #include "hibernate.h" #include "keypin.h" #include "main.h" @@ -33,6 +32,9 @@ #include "routerparse.h" #include "routerset.h" #include "torcert.h" +#include "voting_schedule.h" + +#include "dirauth/dirvote.h" /** * \file dirserv.c @@ -86,7 +88,6 @@ static const signed_descriptor_t *get_signed_descriptor_by_fp( int extrainfo); static was_router_added_t dirserv_add_extrainfo(extrainfo_t *ei, const char **msg); -static uint32_t dirserv_get_bandwidth_for_router_kb(const routerinfo_t *ri); static uint32_t dirserv_get_credible_bandwidth_kb(const routerinfo_t *ri); static int spooled_resource_lookup_body(const spooled_resource_t *spooled, @@ -258,11 +259,12 @@ dirserv_load_fingerprint_file(void) * identity to stop doing so. This is going to be essential for good identity * security: otherwise anybody who can attack RSA-1024 but not Ed25519 could * just sign fake descriptors missing the Ed25519 key. But we won't actually - * be able to prevent that kind of thing until we're confident that there - * isn't actually a legit reason to downgrade to 0.2.5. So for now, we have - * to leave this #undef. + * be able to prevent that kind of thing until we're confident that there isn't + * actually a legit reason to downgrade to 0.2.5. Now we are not recommending + * 0.2.5 anymore so there is no reason to keep the #undef. */ -#undef DISABLE_DISABLING_ED25519 + +#define DISABLE_DISABLING_ED25519 /** Check whether <b>router</b> has a nickname/identity key combination that * we recognize from the fingerprint list, or an IP we automatically act on @@ -857,13 +859,13 @@ directory_remove_invalid(void) SMARTLIST_FOREACH_BEGIN(nodes, node_t *, node) { const char *msg = NULL; + const char *description; routerinfo_t *ent = node->ri; - char description[NODE_DESC_BUF_LEN]; uint32_t r; if (!ent) continue; r = dirserv_router_get_status(ent, &msg, LOG_INFO); - router_get_description(description, ent); + description = router_describe(ent); if (r & FP_REJECT) { log_info(LD_DIRSERV, "Router %s is now rejected: %s", description, msg?msg:""); @@ -920,7 +922,7 @@ list_single_server_status(const routerinfo_t *desc, int is_live) } /* DOCDOC running_long_enough_to_decide_unreachable */ -static inline int +int running_long_enough_to_decide_unreachable(void) { return time_of_process_start @@ -953,7 +955,7 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now) tor_assert(node); if (router_is_me(router)) { - /* We always know if we are down ourselves. */ + /* We always know if we are shutting down or hibernating ourselves. */ answer = ! we_are_hibernating(); } else if (router->is_hibernating && (router->cache_info.published_on + @@ -1055,59 +1057,6 @@ list_server_status_v1(smartlist_t *routers, char **router_status_out, return 0; } -/** Given a (possibly empty) list of config_line_t, each line of which contains - * a list of comma-separated version numbers surrounded by optional space, - * allocate and return a new string containing the version numbers, in order, - * separated by commas. Used to generate Recommended(Client|Server)?Versions - */ -char * -format_recommended_version_list(const config_line_t *ln, int warn) -{ - smartlist_t *versions; - char *result; - versions = smartlist_new(); - for ( ; ln; ln = ln->next) { - smartlist_split_string(versions, ln->value, ",", - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - } - - /* Handle the case where a dirauth operator has accidentally made some - * versions space-separated instead of comma-separated. */ - smartlist_t *more_versions = smartlist_new(); - SMARTLIST_FOREACH_BEGIN(versions, char *, v) { - if (strchr(v, ' ')) { - if (warn) - log_warn(LD_DIRSERV, "Unexpected space in versions list member %s. " - "(These are supposed to be comma-separated; I'll pretend you " - "used commas instead.)", escaped(v)); - SMARTLIST_DEL_CURRENT(versions, v); - smartlist_split_string(more_versions, v, NULL, - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - tor_free(v); - } - } SMARTLIST_FOREACH_END(v); - smartlist_add_all(versions, more_versions); - smartlist_free(more_versions); - - /* Check to make sure everything looks like a version. */ - if (warn) { - SMARTLIST_FOREACH_BEGIN(versions, const char *, v) { - tor_version_t ver; - if (tor_version_parse(v, &ver) < 0) { - log_warn(LD_DIRSERV, "Recommended version %s does not look valid. " - " (I'll include it anyway, since you told me to.)", - escaped(v)); - } - } SMARTLIST_FOREACH_END(v); - } - - sort_version_list(versions, 1); - result = smartlist_join_strings(versions,",",0,NULL); - SMARTLIST_FOREACH(versions,char *,s,tor_free(s)); - smartlist_free(versions); - return result; -} - /** Return 1 if <b>ri</b>'s descriptor is "active" -- running, valid, * not hibernating, having observed bw greater 0, and not too old. Else * return 0. @@ -1453,7 +1402,7 @@ dirserv_thinks_router_is_hs_dir(const routerinfo_t *router, * tests aren't instant. If we haven't been running long enough, * trust the relay. */ - if (stats_n_seconds_working > + if (get_uptime() > get_options()->MinUptimeHidServDirectoryV2 * 1.1) uptime = MIN(rep_hist_get_uptime(router->cache_info.identity_digest, now), real_uptime(router, now)); @@ -1497,6 +1446,24 @@ router_counts_toward_thresholds(const node_t *node, time_t now, (have_mbw || !require_mbw); } +/** Look through the routerlist, and using the measured bandwidth cache count + * how many measured bandwidths we know. This is used to decide whether we + * ever trust advertised bandwidths for purposes of assigning flags. */ +void +dirserv_count_measured_bws(const smartlist_t *routers) +{ + /* Initialize this first */ + routers_with_measured_bw = 0; + + /* Iterate over the routerlist and count measured bandwidths */ + SMARTLIST_FOREACH_BEGIN(routers, const routerinfo_t *, ri) { + /* Check if we know a measured bandwidth for this one */ + if (dirserv_has_measured_bw(ri->cache_info.identity_digest)) { + ++routers_with_measured_bw; + } + } SMARTLIST_FOREACH_END(ri); +} + /** Look through the routerlist, the Mean Time Between Failure history, and * the Weighted Fractional Uptime history, and use them to set thresholds for * the Stable, Fast, and Guard flags. Update the fields stable_uptime, @@ -1504,7 +1471,7 @@ router_counts_toward_thresholds(const node_t *node, time_t now, * guard_bandwidth_including_exits, and guard_bandwidth_excluding_exits. * * Also, set the is_exit flag of each router appropriately. */ -static void +void dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil) { int n_active, n_active_nonexit, n_familiar; @@ -1735,7 +1702,7 @@ dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line, } /** Clear and free the measured bandwidth cache */ -STATIC void +void dirserv_clear_measured_bw_cache(void) { if (mbw_cache) { @@ -1767,18 +1734,10 @@ dirserv_expire_measured_bw_cache(time_t now) } } -/** Get the current size of the measured bandwidth cache */ -STATIC int -dirserv_get_measured_bw_cache_size(void) -{ - if (mbw_cache) return digestmap_size(mbw_cache); - else return 0; -} - /** Query the cache by identity digest, return value indicates whether * we found it. The bw_out and as_of_out pointers receive the cached * bandwidth value and the time it was cached if not NULL. */ -STATIC int +int dirserv_query_measured_bw_cache_kb(const char *node_id, long *bw_kb_out, time_t *as_of_out) { @@ -1799,61 +1758,18 @@ dirserv_query_measured_bw_cache_kb(const char *node_id, long *bw_kb_out, } /** Predicate wrapper for dirserv_query_measured_bw_cache() */ -STATIC int +int dirserv_has_measured_bw(const char *node_id) { return dirserv_query_measured_bw_cache_kb(node_id, NULL, NULL); } -/** Get the best estimate of a router's bandwidth for dirauth purposes, - * preferring measured to advertised values if available. */ - -static uint32_t -dirserv_get_bandwidth_for_router_kb(const routerinfo_t *ri) -{ - uint32_t bw_kb = 0; - /* - * Yeah, measured bandwidths in measured_bw_line_t are (implicitly - * signed) longs and the ones router_get_advertised_bandwidth() returns - * are uint32_t. - */ - long mbw_kb = 0; - - if (ri) { - /* - * * First try to see if we have a measured bandwidth; don't bother with - * as_of_out here, on the theory that a stale measured bandwidth is still - * better to trust than an advertised one. - */ - if (dirserv_query_measured_bw_cache_kb(ri->cache_info.identity_digest, - &mbw_kb, NULL)) { - /* Got one! */ - bw_kb = (uint32_t)mbw_kb; - } else { - /* If not, fall back to advertised */ - bw_kb = router_get_advertised_bandwidth(ri) / 1000; - } - } - - return bw_kb; -} - -/** Look through the routerlist, and using the measured bandwidth cache count - * how many measured bandwidths we know. This is used to decide whether we - * ever trust advertised bandwidths for purposes of assigning flags. */ -static void -dirserv_count_measured_bws(const smartlist_t *routers) +/** Get the current size of the measured bandwidth cache */ +int +dirserv_get_measured_bw_cache_size(void) { - /* Initialize this first */ - routers_with_measured_bw = 0; - - /* Iterate over the routerlist and count measured bandwidths */ - SMARTLIST_FOREACH_BEGIN(routers, const routerinfo_t *, ri) { - /* Check if we know a measured bandwidth for this one */ - if (dirserv_has_measured_bw(ri->cache_info.identity_digest)) { - ++routers_with_measured_bw; - } - } SMARTLIST_FOREACH_END(ri); + if (mbw_cache) return digestmap_size(mbw_cache); + else return 0; } /** Return the bandwidth we believe for assigning flags; prefer measured @@ -1916,26 +1832,6 @@ dirserv_get_flag_thresholds_line(void) return result; } -/** Given a platform string as in a routerinfo_t (possibly null), return a - * newly allocated version string for a networkstatus document, or NULL if the - * platform doesn't give a Tor version. */ -static char * -version_from_platform(const char *platform) -{ - if (platform && !strcmpstart(platform, "Tor ")) { - const char *eos = find_whitespace(platform+4); - if (eos && !strcmpstart(eos, " (r")) { - /* XXXX Unify this logic with the other version extraction - * logic in routerparse.c. */ - eos = find_whitespace(eos+1); - } - if (eos) { - return tor_strndup(platform, eos-platform); - } - } - return NULL; -} - /** Helper: write the router-status information in <b>rs</b> into a newly * allocated character buffer. Use the same format as in network-status * documents. If <b>version</b> is non-NULL, add a "v" line for the platform. @@ -2124,145 +2020,6 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version, return result; } -/** Helper for sorting: compares two routerinfos first by address, and then by - * descending order of "usefulness". (An authority is more useful than a - * non-authority; a running router is more useful than a non-running router; - * and a router with more bandwidth is more useful than one with less.) - **/ -static int -compare_routerinfo_by_ip_and_bw_(const void **a, const void **b) -{ - routerinfo_t *first = *(routerinfo_t **)a, *second = *(routerinfo_t **)b; - int first_is_auth, second_is_auth; - uint32_t bw_kb_first, bw_kb_second; - const node_t *node_first, *node_second; - int first_is_running, second_is_running; - - /* we return -1 if first should appear before second... that is, - * if first is a better router. */ - if (first->addr < second->addr) - return -1; - else if (first->addr > second->addr) - return 1; - - /* Potentially, this next bit could cause k n lg n memeq calls. But in - * reality, we will almost never get here, since addresses will usually be - * different. */ - - first_is_auth = - router_digest_is_trusted_dir(first->cache_info.identity_digest); - second_is_auth = - router_digest_is_trusted_dir(second->cache_info.identity_digest); - - if (first_is_auth && !second_is_auth) - return -1; - else if (!first_is_auth && second_is_auth) - return 1; - - node_first = node_get_by_id(first->cache_info.identity_digest); - node_second = node_get_by_id(second->cache_info.identity_digest); - first_is_running = node_first && node_first->is_running; - second_is_running = node_second && node_second->is_running; - - if (first_is_running && !second_is_running) - return -1; - else if (!first_is_running && second_is_running) - return 1; - - bw_kb_first = dirserv_get_bandwidth_for_router_kb(first); - bw_kb_second = dirserv_get_bandwidth_for_router_kb(second); - - if (bw_kb_first > bw_kb_second) - return -1; - else if (bw_kb_first < bw_kb_second) - return 1; - - /* They're equal! Compare by identity digest, so there's a - * deterministic order and we avoid flapping. */ - return fast_memcmp(first->cache_info.identity_digest, - second->cache_info.identity_digest, - DIGEST_LEN); -} - -/** Given a list of routerinfo_t in <b>routers</b>, return a new digestmap_t - * whose keys are the identity digests of those routers that we're going to - * exclude for Sybil-like appearance. */ -static digestmap_t * -get_possible_sybil_list(const smartlist_t *routers) -{ - const or_options_t *options = get_options(); - digestmap_t *omit_as_sybil; - smartlist_t *routers_by_ip = smartlist_new(); - uint32_t last_addr; - int addr_count; - /* Allow at most this number of Tor servers on a single IP address, ... */ - int max_with_same_addr = options->AuthDirMaxServersPerAddr; - if (max_with_same_addr <= 0) - max_with_same_addr = INT_MAX; - - smartlist_add_all(routers_by_ip, routers); - smartlist_sort(routers_by_ip, compare_routerinfo_by_ip_and_bw_); - omit_as_sybil = digestmap_new(); - - last_addr = 0; - addr_count = 0; - SMARTLIST_FOREACH_BEGIN(routers_by_ip, routerinfo_t *, ri) { - if (last_addr != ri->addr) { - last_addr = ri->addr; - addr_count = 1; - } else if (++addr_count > max_with_same_addr) { - digestmap_set(omit_as_sybil, ri->cache_info.identity_digest, ri); - } - } SMARTLIST_FOREACH_END(ri); - - smartlist_free(routers_by_ip); - return omit_as_sybil; -} - -/** If there are entries in <b>routers</b> with exactly the same ed25519 keys, - * remove the older one. If they are exactly the same age, remove the one - * with the greater descriptor digest. May alter the order of the list. */ -static void -routers_make_ed_keys_unique(smartlist_t *routers) -{ - routerinfo_t *ri2; - digest256map_t *by_ed_key = digest256map_new(); - - SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { - ri->omit_from_vote = 0; - if (ri->cache_info.signing_key_cert == NULL) - continue; /* No ed key */ - const uint8_t *pk = ri->cache_info.signing_key_cert->signing_key.pubkey; - if ((ri2 = digest256map_get(by_ed_key, pk))) { - /* Duplicate; must omit one. Set the omit_from_vote flag in whichever - * one has the earlier published_on. */ - const time_t ri_pub = ri->cache_info.published_on; - const time_t ri2_pub = ri2->cache_info.published_on; - if (ri2_pub < ri_pub || - (ri2_pub == ri_pub && - fast_memcmp(ri->cache_info.signed_descriptor_digest, - ri2->cache_info.signed_descriptor_digest,DIGEST_LEN)<0)) { - digest256map_set(by_ed_key, pk, ri); - ri2->omit_from_vote = 1; - } else { - ri->omit_from_vote = 1; - } - } else { - /* Add to map */ - digest256map_set(by_ed_key, pk, ri); - } - } SMARTLIST_FOREACH_END(ri); - - digest256map_free(by_ed_key, NULL); - - /* Now remove every router where the omit_from_vote flag got set. */ - SMARTLIST_FOREACH_BEGIN(routers, const routerinfo_t *, ri) { - if (ri->omit_from_vote) { - SMARTLIST_DEL_CURRENT(routers, ri); - } - } SMARTLIST_FOREACH_END(ri); -} - /** Extract status information from <b>ri</b> and from other authority * functions and store it in <b>rs</b>. <b>rs</b> is zeroed out before it is * set. @@ -2375,25 +2132,6 @@ dirserv_set_routerstatus_testing(routerstatus_t *rs) } } -/** Routerstatus <b>rs</b> is part of a group of routers that are on - * too narrow an IP-space. Clear out its flags since we don't want it be used - * because of its Sybil-like appearance. - * - * Leave its BadExit flag alone though, since if we think it's a bad exit, - * we want to vote that way in case all the other authorities are voting - * Running and Exit. - */ -static void -clear_status_flags_on_sybil(routerstatus_t *rs) -{ - rs->is_authority = rs->is_exit = rs->is_stable = rs->is_fast = - rs->is_flagged_running = rs->is_named = rs->is_valid = - rs->is_hs_dir = rs->is_v2_dir = rs->is_possible_guard = 0; - /* FFFF we might want some mechanism to check later on if we - * missed zeroing any flags: it's easy to add a new flag but - * forget to add it to this clause. */ -} - /** The guardfraction of the guard with identity fingerprint <b>guard_id</b> * is <b>guardfraction_percentage</b>. See if we have a vote routerstatus for * this guard in <b>vote_routerstatuses</b>, and if we do, register the @@ -2710,17 +2448,38 @@ dirserv_read_guardfraction_file(const char *fname, /** * Helper function to parse out a line in the measured bandwidth file - * into a measured_bw_line_t output structure. Returns -1 on failure - * or 0 on success. + * into a measured_bw_line_t output structure. + * + * If <b>line_is_after_headers</b> is true, then if we encounter an incomplete + * bw line, return -1 and warn, since we are after the headers and we should + * only parse bw lines. Return 0 otherwise. + * + * If <b>line_is_after_headers</b> is false then it means that we are not past + * the header block yet. If we encounter an incomplete bw line, return -1 but + * don't warn since there could be additional header lines coming. If we + * encounter a proper bw line, return 0 (and we got past the headers). */ STATIC int -measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line) +measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line, + int line_is_after_headers) { char *line = tor_strdup(orig_line); char *cp = line; int got_bw = 0; int got_node_id = 0; char *strtok_state; /* lame sauce d'jour */ + + if (strlen(line) == 0) { + log_warn(LD_DIRSERV, "Empty line in bandwidth file"); + tor_free(line); + return -1; + } + + /* Remove end of line character, so that is not part of the token */ + if (line[strlen(line) - 1] == '\n') { + line[strlen(line) - 1] = '\0'; + } + cp = tor_strtok_r(cp, " \t", &strtok_state); if (!cp) { @@ -2782,6 +2541,13 @@ measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line) if (got_bw && got_node_id) { tor_free(line); return 0; + } else if (line_is_after_headers == 0) { + /* There could be additional header lines, therefore do not give warnings + * but returns -1 since it's not a complete bw line. */ + log_debug(LD_DIRSERV, "Missing bw or node_id in bandwidth file line: %s", + escaped(orig_line)); + tor_free(line); + return -1; } else { log_warn(LD_DIRSERV, "Incomplete line in bandwidth file: %s", escaped(orig_line)); @@ -2830,6 +2596,11 @@ dirserv_read_measured_bandwidths(const char *from_file, int applied_lines = 0; time_t file_time, now; int ok; + /* This flag will be 1 only when the first successful bw measurement line + * has been encountered, so that measured_bw_line_parse don't give warnings + * if there are additional header lines, as introduced in Bandwidth List spec + * version 1.1.0 */ + int line_is_after_headers = 0; /* Initialise line, so that we can't possibly run off the end. */ memset(line, 0, sizeof(line)); @@ -2877,7 +2648,11 @@ dirserv_read_measured_bandwidths(const char *from_file, while (!feof(fp)) { measured_bw_line_t parsed_line; if (fgets(line, sizeof(line), fp) && strlen(line)) { - if (measured_bw_line_parse(&parsed_line, line) != -1) { + if (measured_bw_line_parse(&parsed_line, line, + line_is_after_headers) != -1) { + /* This condition will be true when the first complete valid bw line + * has been encountered, which means the end of the header lines. */ + line_is_after_headers = 1; /* Also cache the line for dirserv_get_bandwidth_for_router() */ dirserv_cache_measured_bw(&parsed_line, file_time); if (measured_bw_line_apply(&parsed_line, routerstatuses) > 0) @@ -2896,294 +2671,6 @@ dirserv_read_measured_bandwidths(const char *from_file, return 0; } -/** Return a new networkstatus_t* containing our current opinion. (For v3 - * authorities) */ -networkstatus_t * -dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key, - authority_cert_t *cert) -{ - const or_options_t *options = get_options(); - networkstatus_t *v3_out = NULL; - uint32_t addr; - char *hostname = NULL, *client_versions = NULL, *server_versions = NULL; - const char *contact; - smartlist_t *routers, *routerstatuses; - char identity_digest[DIGEST_LEN]; - char signing_key_digest[DIGEST_LEN]; - int listbadexits = options->AuthDirListBadExits; - routerlist_t *rl = router_get_routerlist(); - time_t now = time(NULL); - time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH; - networkstatus_voter_info_t *voter = NULL; - vote_timing_t timing; - digestmap_t *omit_as_sybil = NULL; - const int vote_on_reachability = running_long_enough_to_decide_unreachable(); - smartlist_t *microdescriptors = NULL; - - tor_assert(private_key); - tor_assert(cert); - - if (crypto_pk_get_digest(private_key, signing_key_digest)<0) { - log_err(LD_BUG, "Error computing signing key digest"); - return NULL; - } - if (crypto_pk_get_digest(cert->identity_key, identity_digest)<0) { - log_err(LD_BUG, "Error computing identity key digest"); - return NULL; - } - if (resolve_my_address(LOG_WARN, options, &addr, NULL, &hostname)<0) { - log_warn(LD_NET, "Couldn't resolve my hostname"); - return NULL; - } - if (!hostname || !strchr(hostname, '.')) { - tor_free(hostname); - hostname = tor_dup_ip(addr); - } - - if (options->VersioningAuthoritativeDir) { - client_versions = - format_recommended_version_list(options->RecommendedClientVersions, 0); - server_versions = - format_recommended_version_list(options->RecommendedServerVersions, 0); - } - - contact = get_options()->ContactInfo; - if (!contact) - contact = "(none)"; - - /* - * Do this so dirserv_compute_performance_thresholds() and - * set_routerstatus_from_routerinfo() see up-to-date bandwidth info. - */ - if (options->V3BandwidthsFile) { - dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL); - } else { - /* - * No bandwidths file; clear the measured bandwidth cache in case we had - * one last time around. - */ - if (dirserv_get_measured_bw_cache_size() > 0) { - dirserv_clear_measured_bw_cache(); - } - } - - /* precompute this part, since we need it to decide what "stable" - * means. */ - SMARTLIST_FOREACH(rl->routers, routerinfo_t *, ri, { - dirserv_set_router_is_running(ri, now); - }); - - routers = smartlist_new(); - smartlist_add_all(routers, rl->routers); - routers_make_ed_keys_unique(routers); - /* After this point, don't use rl->routers; use 'routers' instead. */ - routers_sort_by_identity(routers); - omit_as_sybil = get_possible_sybil_list(routers); - - DIGESTMAP_FOREACH(omit_as_sybil, sybil_id, void *, ignore) { - (void) ignore; - rep_hist_make_router_pessimal(sybil_id, now); - } DIGESTMAP_FOREACH_END; - - /* Count how many have measured bandwidths so we know how to assign flags; - * this must come before dirserv_compute_performance_thresholds() */ - dirserv_count_measured_bws(routers); - - dirserv_compute_performance_thresholds(omit_as_sybil); - - routerstatuses = smartlist_new(); - microdescriptors = smartlist_new(); - - SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) { - /* If it has a protover list and contains a protocol name greater than - * MAX_PROTOCOL_NAME_LENGTH, skip it. */ - if (ri->protocol_list && - protover_contains_long_protocol_names(ri->protocol_list)) { - continue; - } - if (ri->cache_info.published_on >= cutoff) { - routerstatus_t *rs; - vote_routerstatus_t *vrs; - node_t *node = node_get_mutable_by_id(ri->cache_info.identity_digest); - if (!node) - continue; - - vrs = tor_malloc_zero(sizeof(vote_routerstatus_t)); - rs = &vrs->status; - set_routerstatus_from_routerinfo(rs, node, ri, now, - listbadexits); - - if (ri->cache_info.signing_key_cert) { - memcpy(vrs->ed25519_id, - ri->cache_info.signing_key_cert->signing_key.pubkey, - ED25519_PUBKEY_LEN); - } - - if (digestmap_get(omit_as_sybil, ri->cache_info.identity_digest)) - clear_status_flags_on_sybil(rs); - - if (!vote_on_reachability) - rs->is_flagged_running = 0; - - vrs->version = version_from_platform(ri->platform); - if (ri->protocol_list) { - vrs->protocols = tor_strdup(ri->protocol_list); - } else { - vrs->protocols = tor_strdup( - protover_compute_for_old_tor(vrs->version)); - } - vrs->microdesc = dirvote_format_all_microdesc_vote_lines(ri, now, - microdescriptors); - - smartlist_add(routerstatuses, vrs); - } - } SMARTLIST_FOREACH_END(ri); - - { - smartlist_t *added = - microdescs_add_list_to_cache(get_microdesc_cache(), - microdescriptors, SAVED_NOWHERE, 0); - smartlist_free(added); - smartlist_free(microdescriptors); - } - - smartlist_free(routers); - digestmap_free(omit_as_sybil, NULL); - - /* Apply guardfraction information to routerstatuses. */ - if (options->GuardfractionFile) { - dirserv_read_guardfraction_file(options->GuardfractionFile, - routerstatuses); - } - - /* This pass through applies the measured bw lines to the routerstatuses */ - if (options->V3BandwidthsFile) { - dirserv_read_measured_bandwidths(options->V3BandwidthsFile, - routerstatuses); - } else { - /* - * No bandwidths file; clear the measured bandwidth cache in case we had - * one last time around. - */ - if (dirserv_get_measured_bw_cache_size() > 0) { - dirserv_clear_measured_bw_cache(); - } - } - - v3_out = tor_malloc_zero(sizeof(networkstatus_t)); - - v3_out->type = NS_TYPE_VOTE; - dirvote_get_preferred_voting_intervals(&timing); - v3_out->published = now; - { - char tbuf[ISO_TIME_LEN+1]; - networkstatus_t *current_consensus = - networkstatus_get_live_consensus(now); - long last_consensus_interval; /* only used to pick a valid_after */ - if (current_consensus) - last_consensus_interval = current_consensus->fresh_until - - current_consensus->valid_after; - else - last_consensus_interval = options->TestingV3AuthInitialVotingInterval; - v3_out->valid_after = - dirvote_get_start_of_next_interval(now, (int)last_consensus_interval, - options->TestingV3AuthVotingStartOffset); - format_iso_time(tbuf, v3_out->valid_after); - log_notice(LD_DIR,"Choosing valid-after time in vote as %s: " - "consensus_set=%d, last_interval=%d", - tbuf, current_consensus?1:0, (int)last_consensus_interval); - } - v3_out->fresh_until = v3_out->valid_after + timing.vote_interval; - v3_out->valid_until = v3_out->valid_after + - (timing.vote_interval * timing.n_intervals_valid); - v3_out->vote_seconds = timing.vote_delay; - v3_out->dist_seconds = timing.dist_delay; - tor_assert(v3_out->vote_seconds > 0); - tor_assert(v3_out->dist_seconds > 0); - tor_assert(timing.n_intervals_valid > 0); - - v3_out->client_versions = client_versions; - v3_out->server_versions = server_versions; - - /* These are hardwired, to avoid disaster. */ - v3_out->recommended_relay_protocols = - tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 " - "Link=4 Microdesc=1-2 Relay=2"); - v3_out->recommended_client_protocols = - tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 " - "Link=4 Microdesc=1-2 Relay=2"); - v3_out->required_client_protocols = - tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 " - "Link=4 Microdesc=1-2 Relay=2"); - v3_out->required_relay_protocols = - tor_strdup("Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 " - "Link=3-4 Microdesc=1 Relay=1-2"); - - /* We are not allowed to vote to require anything we don't have. */ - tor_assert(protover_all_supported(v3_out->required_relay_protocols, NULL)); - tor_assert(protover_all_supported(v3_out->required_client_protocols, NULL)); - - /* We should not recommend anything we don't have. */ - tor_assert_nonfatal(protover_all_supported( - v3_out->recommended_relay_protocols, NULL)); - tor_assert_nonfatal(protover_all_supported( - v3_out->recommended_client_protocols, NULL)); - - v3_out->package_lines = smartlist_new(); - { - config_line_t *cl; - for (cl = get_options()->RecommendedPackages; cl; cl = cl->next) { - if (validate_recommended_package_line(cl->value)) - smartlist_add_strdup(v3_out->package_lines, cl->value); - } - } - - v3_out->known_flags = smartlist_new(); - smartlist_split_string(v3_out->known_flags, - "Authority Exit Fast Guard Stable V2Dir Valid HSDir", - 0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0); - if (vote_on_reachability) - smartlist_add_strdup(v3_out->known_flags, "Running"); - if (listbadexits) - smartlist_add_strdup(v3_out->known_flags, "BadExit"); - smartlist_sort_strings(v3_out->known_flags); - - if (options->ConsensusParams) { - v3_out->net_params = smartlist_new(); - smartlist_split_string(v3_out->net_params, - options->ConsensusParams, NULL, 0, 0); - smartlist_sort_strings(v3_out->net_params); - } - - voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t)); - voter->nickname = tor_strdup(options->Nickname); - memcpy(voter->identity_digest, identity_digest, DIGEST_LEN); - voter->sigs = smartlist_new(); - voter->address = hostname; - voter->addr = addr; - voter->dir_port = router_get_advertised_dir_port(options, 0); - voter->or_port = router_get_advertised_or_port(options); - voter->contact = tor_strdup(contact); - if (options->V3AuthUseLegacyKey) { - authority_cert_t *c = get_my_v3_legacy_cert(); - if (c) { - if (crypto_pk_get_digest(c->identity_key, voter->legacy_id_digest)) { - log_warn(LD_BUG, "Unable to compute digest of legacy v3 identity key"); - memset(voter->legacy_id_digest, 0, DIGEST_LEN); - } - } - } - - v3_out->voters = smartlist_new(); - smartlist_add(v3_out->voters, voter); - v3_out->cert = authority_cert_dup(cert); - v3_out->routerstatus_list = routerstatuses; - /* Note: networkstatus_digest is unset; it won't get set until we actually - * format the vote. */ - - return v3_out; -} - /** As dirserv_get_routerdescs(), but instead of getting signed_descriptor_t * pointers, adds copies of digests to fps_out, and doesn't use the * /tor/server/ prefix. For a /d/ request, adds descriptor digests; for other diff --git a/src/or/dirserv.h b/src/or/dirserv.h index cb7c628387..b243439cc2 100644 --- a/src/or/dirserv.h +++ b/src/or/dirserv.h @@ -157,6 +157,15 @@ void cached_dir_decref(cached_dir_t *d); cached_dir_t *new_cached_dir(char *s, time_t published); char *format_recommended_version_list(const config_line_t *line, int warn); int validate_recommended_package_line(const char *line); +int dirserv_query_measured_bw_cache_kb(const char *node_id, + long *bw_out, + time_t *as_of_out); +void dirserv_clear_measured_bw_cache(void); +int dirserv_has_measured_bw(const char *node_id); +int dirserv_get_measured_bw_cache_size(void); +void dirserv_count_measured_bws(const smartlist_t *routers); +int running_long_enough_to_decide_unreachable(void); +void dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil); #ifdef DIRSERV_PRIVATE @@ -165,20 +174,15 @@ STATIC void dirserv_set_routerstatus_testing(routerstatus_t *rs); /* Put the MAX_MEASUREMENT_AGE #define here so unit tests can see it */ #define MAX_MEASUREMENT_AGE (3*24*60*60) /* 3 days */ -STATIC int measured_bw_line_parse(measured_bw_line_t *out, const char *line); +STATIC int measured_bw_line_parse(measured_bw_line_t *out, const char *line, + int line_is_after_headers); STATIC int measured_bw_line_apply(measured_bw_line_t *parsed_line, smartlist_t *routerstatuses); STATIC void dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line, time_t as_of); -STATIC void dirserv_clear_measured_bw_cache(void); STATIC void dirserv_expire_measured_bw_cache(time_t now); -STATIC int dirserv_get_measured_bw_cache_size(void); -STATIC int dirserv_query_measured_bw_cache_kb(const char *node_id, - long *bw_out, - time_t *as_of_out); -STATIC int dirserv_has_measured_bw(const char *node_id); STATIC int dirserv_read_guardfraction_file_from_str(const char *guardfraction_file_str, diff --git a/src/or/dns.c b/src/or/dns.c index 411e2d5aa6..ba734ed900 100644 --- a/src/or/dns.c +++ b/src/or/dns.c @@ -56,6 +56,7 @@ #include "connection.h" #include "connection_edge.h" #include "control.h" +#include "crypto_rand.h" #include "dns.h" #include "main.h" #include "policies.h" diff --git a/src/or/dos.c b/src/or/dos.c index 2cb3470582..ee731accea 100644 --- a/src/or/dos.c +++ b/src/or/dos.c @@ -11,6 +11,7 @@ #include "or.h" #include "channel.h" #include "config.h" +#include "crypto_rand.h" #include "geoip.h" #include "main.h" #include "networkstatus.h" diff --git a/src/or/entrynodes.c b/src/or/entrynodes.c index 54638810fa..27d760f1a8 100644 --- a/src/or/entrynodes.c +++ b/src/or/entrynodes.c @@ -118,11 +118,13 @@ #include "circpathbias.h" #include "circuitbuild.h" #include "circuitlist.h" +#include "circuituse.h" #include "circuitstats.h" #include "config.h" #include "confparse.h" #include "connection.h" #include "control.h" +#include "crypto_rand.h" #include "directory.h" #include "entrynodes.h" #include "main.h" @@ -432,14 +434,15 @@ get_guard_confirmed_min_lifetime(void) STATIC int get_n_primary_guards(void) { - const int n = get_options()->NumEntryGuards; - const int n_dir = get_options()->NumDirectoryGuards; - if (n > 5) { - return MAX(n_dir, n + n / 2); - } else if (n >= 1) { - return MAX(n_dir, n * 2); + /* If the user has explicitly configured the number of primary guards, do + * what the user wishes to do */ + const int configured_primaries = get_options()->NumPrimaryGuards; + if (configured_primaries) { + return configured_primaries; } + /* otherwise check for consensus parameter and if that's not set either, just + * use the default value. */ return networkstatus_get_param(NULL, "guard-n-primary-guards", DFLT_N_PRIMARY_GUARDS, 1, INT32_MAX); @@ -454,6 +457,9 @@ get_n_primary_guards_to_use(guard_usage_t usage) int configured; const char *param_name; int param_default; + + /* If the user has explicitly configured the amount of guards, use + that. Otherwise, fall back to the default value. */ if (usage == GUARD_USAGE_DIRGUARD) { configured = get_options()->NumDirectoryGuards; param_name = "guard-n-primary-dir-guards-to-use"; @@ -2335,7 +2341,7 @@ entry_guard_cancel(circuit_guard_state_t **guard_state_p) } /** - * Called by the circuit building module when a circuit has succeeded: + * Called by the circuit building module when a circuit has failed: * informs the guards code that the guard in *<b>guard_state_p</b> is * not working, and advances the state of the guard module. */ @@ -3474,12 +3480,18 @@ guards_update_all(void) used. */ const node_t * guards_choose_guard(cpath_build_state_t *state, - circuit_guard_state_t **guard_state_out) + uint8_t purpose, + circuit_guard_state_t **guard_state_out) { const node_t *r = NULL; const uint8_t *exit_id = NULL; entry_guard_restriction_t *rst = NULL; - if (state && (exit_id = build_state_get_exit_rsa_id(state))) { + + /* Only apply restrictions if we have a specific exit node in mind, and only + * if we are not doing vanguard circuits: we don't want to apply guard + * restrictions to vanguard circuits. */ + if (state && !circuit_should_use_vanguards(purpose) && + (exit_id = build_state_get_exit_rsa_id(state))) { /* We're building to a targeted exit node, so that node can't be * chosen as our guard for this circuit. Remember that fact in a * restriction. */ diff --git a/src/or/entrynodes.h b/src/or/entrynodes.h index d562498313..e8c91da41b 100644 --- a/src/or/entrynodes.h +++ b/src/or/entrynodes.h @@ -322,6 +322,7 @@ struct circuit_guard_state_t { /* Common entry points for old and new guard code */ int guards_update_all(void); const node_t *guards_choose_guard(cpath_build_state_t *state, + uint8_t purpose, circuit_guard_state_t **guard_state_out); const node_t *guards_choose_dirguard(uint8_t dir_purpose, circuit_guard_state_t **guard_state_out); diff --git a/src/or/ext_orport.c b/src/or/ext_orport.c index 16a250fa58..b842442caf 100644 --- a/src/or/ext_orport.c +++ b/src/or/ext_orport.c @@ -20,9 +20,11 @@ #include "or.h" #include "connection.h" #include "connection_or.h" -#include "ext_orport.h" #include "control.h" #include "config.h" +#include "crypto_rand.h" +#include "crypto_util.h" +#include "ext_orport.h" #include "main.h" #include "proto_ext_or.h" #include "util.h" diff --git a/src/or/geoip.c b/src/or/geoip.c index 0ff1c6ce0d..d59043a7f6 100644 --- a/src/or/geoip.c +++ b/src/or/geoip.c @@ -150,7 +150,7 @@ geoip_add_entry(const tor_addr_t *low, const tor_addr_t *high, idx = ((uintptr_t)idxplus1_)-1; } { - geoip_country_t *c = smartlist_get(geoip_countries, idx); + geoip_country_t *c = smartlist_get(geoip_countries, (int)idx); tor_assert(!strcasecmp(c->countrycode, country)); } @@ -628,8 +628,7 @@ geoip_note_client_seen(geoip_client_action_t action, /* Only remember statistics if the DoS mitigation subsystem is enabled. If * not, only if as entry guard or as bridge. */ if (!dos_enabled()) { - if (!options->EntryStatistics && - (!(options->BridgeRelay && options->BridgeRecordUsageByCountry))) { + if (!options->EntryStatistics && !should_record_bridge_info(options)) { return; } } @@ -1881,5 +1880,8 @@ geoip_free_all(void) clear_geoip_db(); tor_free(bridge_stats_extrainfo); + + memset(geoip_digest, 0, sizeof(geoip_digest)); + memset(geoip6_digest, 0, sizeof(geoip6_digest)); } diff --git a/src/or/hibernate.c b/src/or/hibernate.c index 4dc35f68d0..d7d259470f 100644 --- a/src/or/hibernate.c +++ b/src/or/hibernate.c @@ -36,6 +36,7 @@ hibernating, phase 2: #include "connection_edge.h" #include "connection_or.h" #include "control.h" +#include "crypto_rand.h" #include "hibernate.h" #include "main.h" #include "router.h" @@ -51,6 +52,10 @@ static time_t hibernate_end_time = 0; * we aren't shutting down. */ static time_t shutdown_time = 0; +/** A timed event that we'll use when it's time to wake up from + * hibernation. */ +static mainloop_event_t *wakeup_event = NULL; + /** Possible accounting periods. */ typedef enum { UNIT_MONTH=1, UNIT_WEEK=2, UNIT_DAY=3, @@ -130,6 +135,8 @@ static time_t start_of_accounting_period_after(time_t now); static time_t start_of_accounting_period_containing(time_t now); static void accounting_set_wakeup_time(void); static void on_hibernate_state_change(hibernate_state_t prev_state); +static void hibernate_schedule_wakeup_event(time_t now, time_t end_time); +static void wakeup_event_callback(mainloop_event_t *ev, void *data); /** * Return the human-readable name for the hibernation state <b>state</b> @@ -297,7 +304,7 @@ accounting_get_end_time,(void)) return interval_end_time; } -/** Called from main.c to tell us that <b>seconds</b> seconds have +/** Called from connection.c to tell us that <b>seconds</b> seconds have * passed, <b>n_read</b> bytes have been read, and <b>n_written</b> * bytes have been written. */ void @@ -866,7 +873,7 @@ hibernate_end(hibernate_state_t new_state) hibernate_state = new_state; hibernate_end_time = 0; /* no longer hibernating */ - stats_n_seconds_working = 0; /* reset published uptime */ + reset_uptime(); /* reset published uptime */ } /** A wrapper around hibernate_begin, for when we get SIGINT. */ @@ -876,13 +883,26 @@ hibernate_begin_shutdown(void) hibernate_begin(HIBERNATE_STATE_EXITING, time(NULL)); } -/** Return true iff we are currently hibernating. */ +/** + * Return true iff we are currently hibernating -- that is, if we are in + * any non-live state. + */ MOCK_IMPL(int, we_are_hibernating,(void)) { return hibernate_state != HIBERNATE_STATE_LIVE; } +/** + * Return true iff we are currently _fully_ hibernating -- that is, if we are + * in a state where we expect to handle no network activity at all. + */ +MOCK_IMPL(int, +we_are_fully_hibernating,(void)) +{ + return hibernate_state == HIBERNATE_STATE_DORMANT; +} + /** If we aren't currently dormant, close all connections and become * dormant. */ static void @@ -935,6 +955,63 @@ hibernate_go_dormant(time_t now) or_state_mark_dirty(get_or_state(), get_options()->AvoidDiskWrites ? now+600 : 0); + + hibernate_schedule_wakeup_event(now, hibernate_end_time); +} + +/** + * Schedule a mainloop event at <b>end_time</b> to wake up from a dormant + * state. We can't rely on this happening from second_elapsed_callback, + * since second_elapsed_callback will be shut down when we're dormant. + * + * (Note that We might immediately go back to sleep after we set the next + * wakeup time.) + */ +static void +hibernate_schedule_wakeup_event(time_t now, time_t end_time) +{ + struct timeval delay = { 0, 0 }; + + if (now >= end_time) { + // In these cases we always wait at least a second, to avoid running + // the callback in a tight loop. + delay.tv_sec = 1; + } else { + delay.tv_sec = (end_time - now); + } + + if (!wakeup_event) { + wakeup_event = mainloop_event_postloop_new(wakeup_event_callback, NULL); + } + + mainloop_event_schedule(wakeup_event, &delay); +} + +/** + * Called at the end of the interval, or at the wakeup time of the current + * interval, to exit the dormant state. + **/ +static void +wakeup_event_callback(mainloop_event_t *ev, void *data) +{ + (void) ev; + (void) data; + + const time_t now = time(NULL); + accounting_run_housekeeping(now); + consider_hibernation(now); + if (hibernate_state != HIBERNATE_STATE_DORMANT) { + /* We woke up, so everything's great here */ + return; + } + + /* We're still dormant. */ + if (now < interval_wakeup_time) + hibernate_end_time = interval_wakeup_time; + else + hibernate_end_time = interval_end_time; + + hibernate_schedule_wakeup_event(now, hibernate_end_time); } /** Called when hibernate_end_time has arrived. */ @@ -1111,10 +1188,30 @@ getinfo_helper_accounting(control_connection_t *conn, static void on_hibernate_state_change(hibernate_state_t prev_state) { - (void)prev_state; /* Should we do something with this? */ control_event_server_status(LOG_NOTICE, "HIBERNATION_STATUS STATUS=%s", hibernate_state_to_string(hibernate_state)); + + /* We are changing hibernation state, this can affect the main loop event + * list. Rescan it to update the events state. We do this whatever the new + * hibernation state because they can each possibly affect an event. The + * initial state means we are booting up so we shouldn't scan here because + * at this point the events in the list haven't been initialized. */ + if (prev_state != HIBERNATE_STATE_INITIAL) { + rescan_periodic_events(get_options()); + } + + reschedule_per_second_timer(); +} + +/** Free all resources held by the accounting module */ +void +accounting_free_all(void) +{ + mainloop_event_free(wakeup_event); + hibernate_state = HIBERNATE_STATE_INITIAL; + hibernate_end_time = 0; + shutdown_time = 0; } #ifdef TOR_UNIT_TESTS diff --git a/src/or/hibernate.h b/src/or/hibernate.h index 85fb42864b..453969d052 100644 --- a/src/or/hibernate.h +++ b/src/or/hibernate.h @@ -25,11 +25,13 @@ void accounting_add_bytes(size_t n_read, size_t n_written, int seconds); int accounting_record_bandwidth_usage(time_t now, or_state_t *state); void hibernate_begin_shutdown(void); MOCK_DECL(int, we_are_hibernating, (void)); +MOCK_DECL(int, we_are_fully_hibernating,(void)); void consider_hibernation(time_t now); int getinfo_helper_accounting(control_connection_t *conn, const char *question, char **answer, const char **errmsg); uint64_t get_accounting_max_total(void); +void accounting_free_all(void); #ifdef HIBERNATE_PRIVATE /** Possible values of hibernate_state */ diff --git a/src/or/hs_cache.c b/src/or/hs_cache.c index df53efd32d..ecc845d17f 100644 --- a/src/or/hs_cache.c +++ b/src/or/hs_cache.c @@ -11,6 +11,7 @@ #include "or.h" #include "config.h" +#include "crypto_util.h" #include "hs_ident.h" #include "hs_common.h" #include "hs_client.h" diff --git a/src/or/hs_cache.h b/src/or/hs_cache.h index a141634cc4..0d0085ffdc 100644 --- a/src/or/hs_cache.h +++ b/src/or/hs_cache.h @@ -11,7 +11,6 @@ #include <stdint.h> -#include "crypto.h" #include "crypto_ed25519.h" #include "hs_common.h" #include "hs_descriptor.h" diff --git a/src/or/hs_cell.c b/src/or/hs_cell.c index 5244cfa3dd..03273a44f9 100644 --- a/src/or/hs_cell.c +++ b/src/or/hs_cell.c @@ -8,6 +8,7 @@ #include "or.h" #include "config.h" +#include "crypto_util.h" #include "rendservice.h" #include "replaycache.h" #include "util.h" @@ -369,7 +370,7 @@ introduce1_encrypt_and_encode(trn_cell_introduce1_t *cell, crypto_cipher_free(cipher); offset += encoded_enc_cell_len; /* Compute MAC from the above and put it in the buffer. This function will - * make the adjustment to the encryptled_len to ommit the MAC length. */ + * make the adjustment to the encrypted_len to omit the MAC length. */ compute_introduce_mac(encoded_cell, encoded_cell_len, encrypted, encrypted_len, keys.mac_key, sizeof(keys.mac_key), diff --git a/src/or/hs_circuit.c b/src/or/hs_circuit.c index 0a9999a190..d911f5fde4 100644 --- a/src/or/hs_circuit.c +++ b/src/or/hs_circuit.c @@ -13,6 +13,8 @@ #include "circuitlist.h" #include "circuituse.h" #include "config.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "nodelist.h" #include "policies.h" #include "relay.h" @@ -193,11 +195,8 @@ register_intro_circ(const hs_service_intro_point_t *ip, tor_assert(circ); if (ip->base.is_only_legacy) { - uint8_t digest[DIGEST_LEN]; - if (BUG(crypto_pk_get_digest(ip->legacy_key, (char *) digest) < 0)) { - return; - } - hs_circuitmap_register_intro_circ_v2_service_side(circ, digest); + hs_circuitmap_register_intro_circ_v2_service_side(circ, + ip->legacy_key_digest); } else { hs_circuitmap_register_intro_circ_v3_service_side(circ, &ip->auth_key_kp.pubkey); @@ -683,22 +682,14 @@ setup_introduce1_data(const hs_desc_intro_point_t *ip, origin_circuit_t * hs_circ_service_get_intro_circ(const hs_service_intro_point_t *ip) { - origin_circuit_t *circ = NULL; - tor_assert(ip); if (ip->base.is_only_legacy) { - uint8_t digest[DIGEST_LEN]; - if (BUG(crypto_pk_get_digest(ip->legacy_key, (char *) digest) < 0)) { - goto end; - } - circ = hs_circuitmap_get_intro_circ_v2_service_side(digest); + return hs_circuitmap_get_intro_circ_v2_service_side(ip->legacy_key_digest); } else { - circ = hs_circuitmap_get_intro_circ_v3_service_side( + return hs_circuitmap_get_intro_circ_v3_service_side( &ip->auth_key_kp.pubkey); } - end: - return circ; } /* Called when we fail building a rendezvous circuit at some point other than diff --git a/src/or/hs_circuit.h b/src/or/hs_circuit.h index 2f5beaa168..f69137e1d5 100644 --- a/src/or/hs_circuit.h +++ b/src/or/hs_circuit.h @@ -10,7 +10,6 @@ #define TOR_HS_CIRCUIT_H #include "or.h" -#include "crypto.h" #include "crypto_ed25519.h" #include "hs_service.h" diff --git a/src/or/hs_client.c b/src/or/hs_client.c index 0cfdec1907..5546746202 100644 --- a/src/or/hs_client.c +++ b/src/or/hs_client.c @@ -9,31 +9,31 @@ #define HS_CLIENT_PRIVATE #include "or.h" -#include "hs_circuit.h" -#include "hs_ident.h" +#include "circpathbias.h" +#include "circuitbuild.h" +#include "circuitlist.h" +#include "circuituse.h" +#include "config.h" +#include "connection.h" #include "connection_edge.h" #include "container.h" -#include "rendclient.h" -#include "hs_descriptor.h" +#include "crypto_rand.h" +#include "crypto_util.h" +#include "directory.h" #include "hs_cache.h" #include "hs_cell.h" -#include "hs_ident.h" -#include "config.h" -#include "directory.h" +#include "hs_circuit.h" #include "hs_client.h" #include "hs_control.h" -#include "router.h" -#include "routerset.h" -#include "circuitlist.h" -#include "circuituse.h" -#include "connection.h" -#include "nodelist.h" -#include "circpathbias.h" -#include "connection.h" +#include "hs_descriptor.h" +#include "hs_ident.h" #include "hs_ntor.h" -#include "circuitbuild.h" #include "networkstatus.h" +#include "nodelist.h" #include "reasons.h" +#include "rendclient.h" +#include "router.h" +#include "routerset.h" /* Return a human-readable string for the client fetch status code. */ static const char * @@ -1439,8 +1439,8 @@ hs_client_desc_has_arrived(const hs_ident_dir_conn_t *ident) * connection is considered "fresh" and can continue without being closed * too early. */ base_conn->timestamp_created = now; - base_conn->timestamp_lastread = now; - base_conn->timestamp_lastwritten = now; + base_conn->timestamp_last_read_allowed = now; + base_conn->timestamp_last_write_allowed = now; /* Change connection's state into waiting for a circuit. */ base_conn->state = AP_CONN_STATE_CIRCUIT_WAIT; diff --git a/src/or/hs_common.c b/src/or/hs_common.c index e586516c80..c42ef0c97b 100644 --- a/src/or/hs_common.c +++ b/src/or/hs_common.c @@ -15,6 +15,8 @@ #include "config.h" #include "circuitbuild.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "networkstatus.h" #include "nodelist.h" #include "hs_cache.h" @@ -28,9 +30,8 @@ #include "rendservice.h" #include "routerset.h" #include "router.h" -#include "routerset.h" -#include "shared_random.h" -#include "shared_random_state.h" +#include "shared_random_client.h" +#include "dirauth/shared_random_state.h" /* Trunnel */ #include "ed25519_cert.h" @@ -104,7 +105,7 @@ compare_digest_to_fetch_hsdir_index(const void *_key, const void **_member) { const char *key = _key; const node_t *node = *_member; - return tor_memcmp(key, node->hsdir_index->fetch, DIGEST256_LEN); + return tor_memcmp(key, node->hsdir_index.fetch, DIGEST256_LEN); } /* Helper function: The key is a digest that we compare to a node_t object @@ -115,7 +116,7 @@ compare_digest_to_store_first_hsdir_index(const void *_key, { const char *key = _key; const node_t *node = *_member; - return tor_memcmp(key, node->hsdir_index->store_first, DIGEST256_LEN); + return tor_memcmp(key, node->hsdir_index.store_first, DIGEST256_LEN); } /* Helper function: The key is a digest that we compare to a node_t object @@ -126,7 +127,7 @@ compare_digest_to_store_second_hsdir_index(const void *_key, { const char *key = _key; const node_t *node = *_member; - return tor_memcmp(key, node->hsdir_index->store_second, DIGEST256_LEN); + return tor_memcmp(key, node->hsdir_index.store_second, DIGEST256_LEN); } /* Helper function: Compare two node_t objects current hsdir_index. */ @@ -135,8 +136,8 @@ compare_node_fetch_hsdir_index(const void **a, const void **b) { const node_t *node1= *a; const node_t *node2 = *b; - return tor_memcmp(node1->hsdir_index->fetch, - node2->hsdir_index->fetch, + return tor_memcmp(node1->hsdir_index.fetch, + node2->hsdir_index.fetch, DIGEST256_LEN); } @@ -146,8 +147,8 @@ compare_node_store_first_hsdir_index(const void **a, const void **b) { const node_t *node1= *a; const node_t *node2 = *b; - return tor_memcmp(node1->hsdir_index->store_first, - node2->hsdir_index->store_first, + return tor_memcmp(node1->hsdir_index.store_first, + node2->hsdir_index.store_first, DIGEST256_LEN); } @@ -157,8 +158,8 @@ compare_node_store_second_hsdir_index(const void **a, const void **b) { const node_t *node1= *a; const node_t *node2 = *b; - return tor_memcmp(node1->hsdir_index->store_second, - node2->hsdir_index->store_second, + return tor_memcmp(node1->hsdir_index.store_second, + node2->hsdir_index.store_second, DIGEST256_LEN); } @@ -1289,18 +1290,15 @@ node_has_hsdir_index(const node_t *node) /* At this point, since the node has a desc, this node must also have an * hsdir index. If not, something went wrong, so BUG out. */ - if (BUG(node->hsdir_index == NULL)) { - return 0; - } - if (BUG(tor_mem_is_zero((const char*)node->hsdir_index->fetch, + if (BUG(tor_mem_is_zero((const char*)node->hsdir_index.fetch, DIGEST256_LEN))) { return 0; } - if (BUG(tor_mem_is_zero((const char*)node->hsdir_index->store_first, + if (BUG(tor_mem_is_zero((const char*)node->hsdir_index.store_first, DIGEST256_LEN))) { return 0; } - if (BUG(tor_mem_is_zero((const char*)node->hsdir_index->store_second, + if (BUG(tor_mem_is_zero((const char*)node->hsdir_index.store_second, DIGEST256_LEN))) { return 0; } @@ -1334,15 +1332,20 @@ hs_get_responsible_hsdirs(const ed25519_public_key_t *blinded_pk, sorted_nodes = smartlist_new(); + /* Make sure we actually have a live consensus */ + networkstatus_t *c = networkstatus_get_live_consensus(approx_time()); + if (!c || smartlist_len(c->routerstatus_list) == 0) { + log_warn(LD_REND, "No live consensus so we can't get the responsible " + "hidden service directories."); + goto done; + } + + /* Ensure the nodelist is fresh, since it contains the HSDir indices. */ + nodelist_ensure_freshness(c); + /* Add every node_t that support HSDir v3 for which we do have a valid * hsdir_index already computed for them for this consensus. */ { - networkstatus_t *c = networkstatus_get_latest_consensus(); - if (!c || smartlist_len(c->routerstatus_list) == 0) { - log_warn(LD_REND, "No valid consensus so we can't get the responsible " - "hidden service directories."); - goto done; - } SMARTLIST_FOREACH_BEGIN(c->routerstatus_list, const routerstatus_t *, rs) { /* Even though this node_t object won't be modified and should be const, * we can't add const object in a smartlist_t. */ diff --git a/src/or/hs_common.h b/src/or/hs_common.h index 83ba1b8599..ef7d5dca2b 100644 --- a/src/or/hs_common.h +++ b/src/or/hs_common.h @@ -156,19 +156,6 @@ typedef struct rend_service_port_config_t { char unix_addr[FLEXIBLE_ARRAY_MEMBER]; } rend_service_port_config_t; -/* Hidden service directory index used in a node_t which is set once we set - * the consensus. */ -typedef struct hsdir_index_t { - /* HSDir index to use when fetching a descriptor. */ - uint8_t fetch[DIGEST256_LEN]; - - /* HSDir index used by services to store their first and second - * descriptor. The first descriptor is chronologically older than the second - * one and uses older TP and SRV values. */ - uint8_t store_first[DIGEST256_LEN]; - uint8_t store_second[DIGEST256_LEN]; -} hsdir_index_t; - void hs_init(void); void hs_free_all(void); diff --git a/src/or/hs_control.c b/src/or/hs_control.c index 87b4e3fca8..6b9b95c6d8 100644 --- a/src/or/hs_control.c +++ b/src/or/hs_control.c @@ -8,6 +8,7 @@ #include "or.h" #include "control.h" +#include "crypto_util.h" #include "hs_common.h" #include "hs_control.h" #include "hs_descriptor.h" @@ -39,9 +40,8 @@ hs_control_desc_event_requested(const ed25519_public_key_t *onion_pk, * can't pick a node without an hsdir_index. */ hsdir_node = node_get_by_id(hsdir_rs->identity_digest); tor_assert(hsdir_node); - tor_assert(hsdir_node->hsdir_index); /* This is a fetch event. */ - hsdir_index = hsdir_node->hsdir_index->fetch; + hsdir_index = hsdir_node->hsdir_index.fetch; /* Trigger the event. */ control_event_hs_descriptor_requested(onion_address, REND_NO_AUTH, diff --git a/src/or/hs_descriptor.c b/src/or/hs_descriptor.c index 0298c37322..7ffa885ca8 100644 --- a/src/or/hs_descriptor.c +++ b/src/or/hs_descriptor.c @@ -59,6 +59,8 @@ #include "ed25519_cert.h" /* Trunnel interface. */ #include "hs_descriptor.h" #include "circuitbuild.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "parsecommon.h" #include "rendcache.h" #include "hs_cache.h" diff --git a/src/or/hs_descriptor.h b/src/or/hs_descriptor.h index 09979410e1..8195c6efbc 100644 --- a/src/or/hs_descriptor.h +++ b/src/or/hs_descriptor.h @@ -16,6 +16,7 @@ #include "container.h" #include "crypto.h" #include "crypto_ed25519.h" +#include "ed25519_cert.h" /* needed for trunnel */ #include "torcert.h" /* Trunnel */ diff --git a/src/or/hs_ident.c b/src/or/hs_ident.c index 0bce2f625b..3603e329d4 100644 --- a/src/or/hs_ident.c +++ b/src/or/hs_ident.c @@ -7,6 +7,7 @@ * subsytem. **/ +#include "crypto_util.h" #include "hs_ident.h" /* Return a newly allocated circuit identifier. The given public key is copied diff --git a/src/or/hs_ident.h b/src/or/hs_ident.h index 91ec389aa4..8f9da30c35 100644 --- a/src/or/hs_ident.h +++ b/src/or/hs_ident.h @@ -21,7 +21,6 @@ #ifndef TOR_HS_IDENT_H #define TOR_HS_IDENT_H -#include "crypto.h" #include "crypto_ed25519.h" #include "hs_common.h" diff --git a/src/or/hs_intropoint.c b/src/or/hs_intropoint.c index 9eaf572510..ee79109ca9 100644 --- a/src/or/hs_intropoint.c +++ b/src/or/hs_intropoint.c @@ -12,7 +12,6 @@ #include "config.h" #include "circuitlist.h" #include "circuituse.h" -#include "config.h" #include "relay.h" #include "rendmid.h" #include "rephist.h" diff --git a/src/or/hs_ntor.c b/src/or/hs_ntor.c index a416bc46c3..809fa83bb8 100644 --- a/src/or/hs_ntor.c +++ b/src/or/hs_ntor.c @@ -25,6 +25,7 @@ */ #include "or.h" +#include "crypto_util.h" #include "hs_ntor.h" /* String constants used by the ntor HS protocol */ diff --git a/src/or/hs_service.c b/src/or/hs_service.c index 4c67aff429..1b6316dabe 100644 --- a/src/or/hs_service.c +++ b/src/or/hs_service.c @@ -15,6 +15,8 @@ #include "circuituse.h" #include "config.h" #include "connection.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "directory.h" #include "main.h" #include "networkstatus.h" @@ -24,14 +26,13 @@ #include "router.h" #include "routerkeys.h" #include "routerlist.h" -#include "shared_random_state.h" +#include "shared_random_client.h" #include "statefile.h" #include "hs_circuit.h" #include "hs_common.h" #include "hs_config.h" #include "hs_control.h" -#include "hs_circuit.h" #include "hs_descriptor.h" #include "hs_ident.h" #include "hs_intropoint.h" @@ -81,6 +82,7 @@ static smartlist_t *hs_service_staging_list; * reupload if needed */ static int consider_republishing_hs_descriptors = 0; +/* Static declaration. */ static void set_descriptor_revision_counter(hs_descriptor_t *hs_desc); static void move_descriptors(hs_service_t *src, hs_service_t *dst); @@ -153,6 +155,12 @@ register_service(hs_service_ht *map, hs_service_t *service) } /* Taking ownership of the object at this point. */ HT_INSERT(hs_service_ht, map, service); + + /* If we just modified the global map, we notify. */ + if (map == hs_service_map) { + hs_service_map_has_changed(); + } + return 0; } @@ -179,6 +187,11 @@ remove_service(hs_service_ht *map, hs_service_t *service) "while removing service %s", escaped(service->config.directory_path)); } + + /* If we just modified the global map, we notify. */ + if (map == hs_service_map) { + hs_service_map_has_changed(); + } } /* Set the default values for a service configuration object <b>c</b>. */ @@ -434,6 +447,10 @@ service_intro_point_new(const extend_info_t *ei, unsigned int is_legacy, if (crypto_pk_generate_key(ip->legacy_key) < 0) { goto err; } + if (crypto_pk_get_digest(ip->legacy_key, + (char *) ip->legacy_key_digest) < 0) { + goto err; + } } if (ei == NULL) { @@ -920,6 +937,11 @@ register_all_services(void) smartlist_clear(hs_service_staging_list); service_free_all(); hs_service_map = new_service_map; + /* We've just register services into the new map and now we've replaced the + * global map with it so we have to notify that the change happened. When + * registering a service, the notify is only triggered if the destination + * map is the global map for which in here it was not. */ + hs_service_map_has_changed(); } /* Write the onion address of a given service to the given filename fname_ in @@ -2295,8 +2317,8 @@ upload_descriptor_to_hsdir(const hs_service_t *service, /* Logging so we know where it was sent. */ { int is_next_desc = (service->desc_next == desc); - const uint8_t *idx = (is_next_desc) ? hsdir->hsdir_index->store_second: - hsdir->hsdir_index->store_first; + const uint8_t *idx = (is_next_desc) ? hsdir->hsdir_index.store_second: + hsdir->hsdir_index.store_first; log_info(LD_REND, "Service %s %s descriptor of revision %" PRIu64 " initiated upload request to %s with index %s", safe_str_client(service->onion_address), @@ -2944,6 +2966,17 @@ service_add_fnames_to_list(const hs_service_t *service, smartlist_t *list) /* Public API */ /* ========== */ +/* This is called everytime the service map (v2 or v3) changes that is if an + * element is added or removed. */ +void +hs_service_map_has_changed(void) +{ + /* If we now have services where previously we had not, we need to enable + * the HS service main loop event. If we changed to having no services, we + * need to disable the event. */ + rescan_periodic_events(get_options()); +} + /* Upload an encoded descriptor in encoded_desc of the given version. This * descriptor is for the service identity_pk and blinded_pk used to setup the * directory connection identifier. It is uploaded to the directory hsdir_rs @@ -3041,6 +3074,12 @@ hs_service_add_ephemeral(ed25519_secret_key_t *sk, smartlist_t *ports, goto err; } + /* Build the onion address for logging purposes but also the control port + * uses it for the HS_DESC event. */ + hs_build_address(&service->keys.identity_pk, + (uint8_t) service->config.version, + service->onion_address); + /* The only way the registration can fail is if the service public key * already exists. */ if (BUG(register_service(hs_service_map, service) < 0)) { @@ -3050,14 +3089,10 @@ hs_service_add_ephemeral(ed25519_secret_key_t *sk, smartlist_t *ports, goto err; } - /* Last step is to build the onion address. */ - hs_build_address(&service->keys.identity_pk, - (uint8_t) service->config.version, - service->onion_address); - *address_out = tor_strdup(service->onion_address); - log_info(LD_CONFIG, "Added ephemeral v3 onion service: %s", safe_str_client(service->onion_address)); + + *address_out = tor_strdup(service->onion_address); ret = RSAE_OKAY; goto end; diff --git a/src/or/hs_service.h b/src/or/hs_service.h index f3cd49e073..911e177702 100644 --- a/src/or/hs_service.h +++ b/src/or/hs_service.h @@ -50,6 +50,9 @@ typedef struct hs_service_intro_point_t { /* Legacy key if that intro point doesn't support v3. This should be used if * the base object legacy flag is set. */ crypto_pk_t *legacy_key; + /* Legacy key SHA1 public key digest. This should be used only if the base + * object legacy flag is set. */ + uint8_t legacy_key_digest[DIGEST_LEN]; /* Amount of INTRODUCE2 cell accepted from this intro point. */ uint64_t introduce2_count; @@ -260,6 +263,7 @@ void hs_service_lists_fnames_for_sandbox(smartlist_t *file_list, int hs_service_set_conn_addr_port(const origin_circuit_t *circ, edge_connection_t *conn); +void hs_service_map_has_changed(void); void hs_service_dir_info_changed(void); void hs_service_run_scheduled_events(time_t now); void hs_service_circuit_has_opened(origin_circuit_t *circ); diff --git a/src/or/hs_stats.c b/src/or/hs_stats.c index 3e183a5bfc..1e2a96945b 100644 --- a/src/or/hs_stats.c +++ b/src/or/hs_stats.c @@ -3,7 +3,7 @@ /** * \file hs_stats.c - * \brief Keeps stats about the activity of our hidden service. + * \brief Keeps stats about the activity of our onion service(s). **/ #include "or.h" @@ -42,14 +42,14 @@ hs_stats_get_n_introduce2_v2_cells(void) return n_introduce2_v2; } -/** Note that we attempted to launch another circuit to a rendezvous point */ +/** Note that we attempted to launch another circuit to a rendezvous point. */ void hs_stats_note_service_rendezvous_launch(void) { n_rendezvous_launches++; } -/** Return the number of rendezvous circuits we have attempted to launch */ +/** Return the number of rendezvous circuits we have attempted to launch. */ uint32_t hs_stats_get_n_rendezvous_launches(void) { diff --git a/src/or/include.am b/src/or/include.am index c1e23dd3d9..59d593a5e9 100644 --- a/src/or/include.am +++ b/src/or/include.am @@ -41,10 +41,8 @@ LIBTOR_A_SOURCES = \ src/or/consdiffmgr.c \ src/or/control.c \ src/or/cpuworker.c \ - src/or/dircollate.c \ src/or/directory.c \ src/or/dirserv.c \ - src/or/dirvote.c \ src/or/dns.c \ src/or/dnsserv.c \ src/or/dos.c \ @@ -76,8 +74,6 @@ LIBTOR_A_SOURCES = \ src/or/onion.c \ src/or/onion_fast.c \ src/or/onion_tap.c \ - src/or/shared_random.c \ - src/or/shared_random_state.c \ src/or/transports.c \ src/or/parsecommon.c \ src/or/periodic.c \ @@ -91,6 +87,7 @@ LIBTOR_A_SOURCES = \ src/or/policies.c \ src/or/reasons.c \ src/or/relay.c \ + src/or/relay_crypto.c \ src/or/rendcache.c \ src/or/rendclient.c \ src/or/rendcommon.c \ @@ -106,15 +103,43 @@ LIBTOR_A_SOURCES = \ src/or/scheduler.c \ src/or/scheduler_kist.c \ src/or/scheduler_vanilla.c \ + src/or/shared_random_client.c \ src/or/statefile.c \ src/or/status.c \ src/or/torcert.c \ src/or/tor_api.c \ + src/or/voting_schedule.c \ src/or/onion_ntor.c \ $(tor_platform_source) +# +# Modules are conditionnally compiled in tor starting here. We add the C files +# only if the modules has been enabled at configure time. We always add the +# source files of every module to libtor-testing.a so we can build the unit +# tests for everything. See the UNITTESTS_ENABLED branch below. +# +LIBTOR_TESTING_A_SOURCES = $(LIBTOR_A_SOURCES) + +# The Directory Authority module. +MODULE_DIRAUTH_SOURCES = \ + src/or/dirauth/dircollate.c \ + src/or/dirauth/dirvote.c \ + src/or/dirauth/shared_random.c \ + src/or/dirauth/shared_random_state.c +if BUILD_MODULE_DIRAUTH +LIBTOR_A_SOURCES += $(MODULE_DIRAUTH_SOURCES) +endif + src_or_libtor_a_SOURCES = $(LIBTOR_A_SOURCES) -src_or_libtor_testing_a_SOURCES = $(LIBTOR_A_SOURCES) +if UNITTESTS_ENABLED + +# Add the sources of the modules that are needed for tests to work here. +LIBTOR_TESTING_A_SOURCES += $(MODULE_DIRAUTH_SOURCES) + +src_or_libtor_testing_a_SOURCES = $(LIBTOR_TESTING_A_SOURCES) +else +src_or_libtor_testing_a_SOURCES = +endif src_or_tor_SOURCES = src/or/tor_main.c AM_CPPFLAGS += -I$(srcdir)/src/or -Isrc/or @@ -184,10 +209,8 @@ ORHEADERS = \ src/or/consdiffmgr.h \ src/or/control.h \ src/or/cpuworker.h \ - src/or/dircollate.h \ src/or/directory.h \ src/or/dirserv.h \ - src/or/dirvote.h \ src/or/dns.h \ src/or/dns_structs.h \ src/or/dnsserv.h \ @@ -224,8 +247,6 @@ ORHEADERS = \ src/or/onion_ntor.h \ src/or/onion_tap.h \ src/or/or.h \ - src/or/shared_random.h \ - src/or/shared_random_state.h \ src/or/transports.h \ src/or/parsecommon.h \ src/or/periodic.h \ @@ -238,6 +259,7 @@ ORHEADERS = \ src/or/proto_socks.h \ src/or/reasons.h \ src/or/relay.h \ + src/or/relay_crypto.h \ src/or/rendcache.h \ src/or/rendclient.h \ src/or/rendcommon.h \ @@ -252,10 +274,23 @@ ORHEADERS = \ src/or/routerset.h \ src/or/routerparse.h \ src/or/scheduler.h \ + src/or/shared_random_client.h \ src/or/statefile.h \ src/or/status.h \ src/or/torcert.h \ - src/or/tor_api_internal.h + src/or/tor_api_internal.h \ + src/or/voting_schedule.h + +# We add the headers of the modules even though they are disabled so we can +# properly compiled the entry points stub. + +# The Directory Authority module headers. +ORHEADERS += \ + src/or/dirauth/dircollate.h \ + src/or/dirauth/dirvote.h \ + src/or/dirauth/mode.h \ + src/or/dirauth/shared_random.h \ + src/or/dirauth/shared_random_state.h # This may someday want to be an installed file? noinst_HEADERS += src/or/tor_api.h diff --git a/src/or/keypin.c b/src/or/keypin.c index 1698dc184f..97e16c1f78 100644 --- a/src/or/keypin.c +++ b/src/or/keypin.c @@ -12,7 +12,7 @@ #include "orconfig.h" #include "compat.h" -#include "crypto.h" +#include "crypto_digest.h" #include "crypto_format.h" #include "di_ops.h" #include "ht.h" @@ -289,8 +289,10 @@ static int keypin_journal_fd = -1; int keypin_open_journal(const char *fname) { - /* O_SYNC ??*/ - int fd = tor_open_cloexec(fname, O_WRONLY|O_CREAT|O_BINARY, 0600); +#ifndef O_SYNC +#define O_SYNC 0 +#endif + int fd = tor_open_cloexec(fname, O_WRONLY|O_CREAT|O_BINARY|O_SYNC, 0600); if (fd < 0) goto err; @@ -417,10 +419,11 @@ keypin_load_journal_impl(const char *data, size_t size) ++n_entries; } - int severity = (n_corrupt_lines || n_duplicates) ? LOG_WARN : LOG_INFO; + int severity = (n_corrupt_lines || n_duplicates) ? LOG_NOTICE : LOG_INFO; tor_log(severity, LD_DIRSERV, "Loaded %d entries from keypin journal. " - "Found %d corrupt lines, %d duplicates, and %d conflicts.", + "Found %d corrupt lines (ignored), %d duplicates (harmless), " + "and %d conflicts (resolved in favor or more recent entry).", n_entries, n_corrupt_lines, n_duplicates, n_conflicts); return 0; diff --git a/src/or/main.c b/src/or/main.c index 3aa730dcec..bc01e07c3d 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -59,6 +59,7 @@ #include "circuitbuild.h" #include "circuitlist.h" #include "circuituse.h" +#include "circuitmux_ewma.h" #include "command.h" #include "compress.h" #include "config.h" @@ -70,9 +71,9 @@ #include "control.h" #include "cpuworker.h" #include "crypto_s2k.h" +#include "crypto_rand.h" #include "directory.h" #include "dirserv.h" -#include "dirvote.h" #include "dns.h" #include "dnsserv.h" #include "dos.h" @@ -103,7 +104,6 @@ #include "routerlist.h" #include "routerparse.h" #include "scheduler.h" -#include "shared_random.h" #include "statefile.h" #include "status.h" #include "tor_api.h" @@ -118,6 +118,10 @@ #include <event2/event.h> +#include "dirauth/dirvote.h" +#include "dirauth/mode.h" +#include "dirauth/shared_random.h" + #ifdef HAVE_SYSTEMD # if defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__) /* Systemd's use of gcc's __INCLUDE_LEVEL__ extension macro appears to confuse @@ -133,7 +137,7 @@ void evdns_shutdown(int); #ifdef HAVE_RUST // helper function defined in Rust to output a log message indicating if tor is // running with Rust enabled. See src/rust/tor_util -char *rust_welcome_string(void); +void rust_log_welcome_string(void); #endif /********* PROTOTYPES **********/ @@ -152,24 +156,12 @@ static void shutdown_did_not_work_callback(evutil_socket_t fd, short event, void *arg) ATTR_NORETURN; /********* START VARIABLES **********/ -int global_read_bucket; /**< Max number of bytes I can read this second. */ -int global_write_bucket; /**< Max number of bytes I can write this second. */ - -/** Max number of relayed (bandwidth class 1) bytes I can read this second. */ -int global_relayed_read_bucket; -/** Max number of relayed (bandwidth class 1) bytes I can write this second. */ -int global_relayed_write_bucket; -/** What was the read bucket before the last second_elapsed_callback() call? - * (used to determine how many bytes we've read). */ -static int stats_prev_global_read_bucket; -/** What was the write bucket before the last second_elapsed_callback() call? - * (used to determine how many bytes we've written). */ -static int stats_prev_global_write_bucket; - -/* DOCDOC stats_prev_n_read */ -static uint64_t stats_prev_n_read = 0; -/* DOCDOC stats_prev_n_written */ -static uint64_t stats_prev_n_written = 0; + +/* Token bucket for all traffic. */ +token_bucket_rw_t global_bucket; + +/* Token bucket for relayed traffic. */ +token_bucket_rw_t global_relayed_bucket; /* XXX we might want to keep stats about global_relayed_*_bucket too. Or not.*/ /** How many bytes have we read since we started the process? */ @@ -179,7 +171,7 @@ static uint64_t stats_n_bytes_written = 0; /** What time did this process start up? */ time_t time_of_process_start = 0; /** How many seconds have we been running? */ -long stats_n_seconds_working = 0; +static long stats_n_seconds_working = 0; /** How many times have we returned from the main loop successfully? */ static uint64_t stats_n_main_loop_successes = 0; /** How many times have we received an error from the main loop? */ @@ -193,6 +185,8 @@ static uint64_t stats_n_main_loop_idle = 0; static time_t time_of_last_signewnym = 0; /** Is there a signewnym request we're currently waiting to handle? */ static int signewnym_is_pending = 0; +/** Mainloop event for the deferred signewnym call. */ +static mainloop_event_t *handle_deferred_signewnym_ev = NULL; /** How many times have we called newnym? */ static unsigned newnym_epoch = 0; @@ -410,6 +404,27 @@ connection_unlink(connection_t *conn) connection_free(conn); } +/** + * Callback: used to activate read events for all linked connections, so + * libevent knows to call their read callbacks. This callback run as a + * postloop event, so that the events _it_ activates don't happen until + * Libevent has a chance to check for other events. + */ +static void +schedule_active_linked_connections_cb(mainloop_event_t *event, void *arg) +{ + (void)event; + (void)arg; + + /* All active linked conns should get their read events activated, + * so that libevent knows to run their callbacks. */ + SMARTLIST_FOREACH(active_linked_connection_lst, connection_t *, conn, + event_active(conn->read_event, EV_READ, 1)); +} + +/** Event that invokes schedule_active_linked_connections_cb. */ +static mainloop_event_t *schedule_active_linked_connections_event = NULL; + /** Initialize the global connection list, closeable connection list, * and active connection list. */ STATIC void @@ -431,6 +446,7 @@ add_connection_to_closeable_list(connection_t *conn) tor_assert(conn->marked_for_close); assert_connection_ok(conn, time(NULL)); smartlist_add(closeable_connection_lst, conn); + mainloop_schedule_postloop_cleanup(); } /** Return 1 if conn is on the closeable list, else return 0. */ @@ -458,21 +474,37 @@ get_connection_array, (void)) return connection_array; } -/** Provides the traffic read and written over the life of the process. */ - +/** + * Return the amount of network traffic read, in bytes, over the life of this + * process. + */ MOCK_IMPL(uint64_t, get_bytes_read,(void)) { return stats_n_bytes_read; } -/* DOCDOC get_bytes_written */ +/** + * Return the amount of network traffic read, in bytes, over the life of this + * process. + */ MOCK_IMPL(uint64_t, get_bytes_written,(void)) { return stats_n_bytes_written; } +/** + * Increment the amount of network traffic read and written, over the life of + * this process. + */ +void +stats_increment_bytes_read_and_written(uint64_t r, uint64_t w) +{ + stats_n_bytes_read += r; + stats_n_bytes_written += w; +} + /** Set the event mask on <b>conn</b> to <b>events</b>. (The event * mask is a bitmask whose bits are READ_EVENT and WRITE_EVENT) */ @@ -710,20 +742,6 @@ connection_should_read_from_linked_conn(connection_t *conn) return 0; } -/** If we called event_base_loop() and told it to never stop until it - * runs out of events, now we've changed our mind: tell it we want it to - * exit once the current round of callbacks is done, so that we can - * run external code, and then return to the main loop. */ -void -tell_event_loop_to_run_external_code(void) -{ - if (!called_loop_once) { - struct timeval tv = { 0, 0 }; - tor_event_base_loopexit(tor_libevent_get_base(), &tv); - called_loop_once = 1; /* hack to avoid adding more exit events */ - } -} - /** Event to run 'shutdown did not work callback'. */ static struct event *shutdown_did_not_work_event = NULL; @@ -779,8 +797,9 @@ tor_shutdown_event_loop_and_exit(int exitcode) shutdown_did_not_work_callback, NULL); event_add(shutdown_did_not_work_event, &ten_seconds); - /* Unlike loopexit, loopbreak prevents other callbacks from running. */ - tor_event_base_loopbreak(tor_libevent_get_base()); + /* Unlike exit_loop_after_delay(), exit_loop_after_callback + * prevents other callbacks from running. */ + tor_libevent_exit_loop_after_callback(tor_libevent_get_base()); } /** Return true iff tor_shutdown_event_loop_and_exit() has been called. */ @@ -802,10 +821,7 @@ connection_start_reading_from_linked_conn(connection_t *conn) if (!conn->active_on_link) { conn->active_on_link = 1; smartlist_add(active_linked_connection_lst, conn); - /* make sure that the event_base_loop() function exits at - * the end of its run through the current connections, so we can - * activate read events for linked connections. */ - tell_event_loop_to_run_external_code(); + mainloop_event_activate(schedule_active_linked_connections_event); } else { tor_assert(smartlist_contains(active_linked_connection_lst, conn)); } @@ -1008,7 +1024,8 @@ conn_close_if_marked(int i) LOG_FN_CONN(conn, (LOG_INFO,LD_NET, "Holding conn (fd %d) open for more flushing.", (int)conn->s)); - conn->timestamp_lastwritten = now; /* reset so we can flush more */ + conn->timestamp_last_write_allowed = now; /* reset so we can flush + * more */ } else if (sz == 0) { /* Also, retval==0. If we get here, we didn't want to write anything * (because of rate-limiting) and we didn't. */ @@ -1019,19 +1036,22 @@ conn_close_if_marked(int i) * busy Libevent loops where we keep ending up here and returning * 0 until we are no longer blocked on bandwidth. */ - if (connection_is_writing(conn)) { - conn->write_blocked_on_bw = 1; - connection_stop_writing(conn); + connection_consider_empty_read_buckets(conn); + connection_consider_empty_write_buckets(conn); + + /* Make sure that consider_empty_buckets really disabled the + * connection: */ + if (BUG(connection_is_writing(conn))) { + connection_write_bw_exhausted(conn, true); } - if (connection_is_reading(conn)) { + if (BUG(connection_is_reading(conn))) { /* XXXX+ We should make this code unreachable; if a connection is * marked for close and flushing, there is no point in reading to it * at all. Further, checking at this point is a bit of a hack: it * would make much more sense to react in * connection_handle_read_impl, or to just stop reading in * mark_and_flush */ - conn->read_blocked_on_bw = 1; - connection_stop_reading(conn); + connection_read_bw_exhausted(conn, true/* kludge. */); } } return 0; @@ -1059,9 +1079,8 @@ conn_close_if_marked(int i) * reason. */ static void -directory_all_unreachable_cb(evutil_socket_t fd, short event, void *arg) +directory_all_unreachable_cb(mainloop_event_t *event, void *arg) { - (void)fd; (void)event; (void)arg; @@ -1081,7 +1100,7 @@ directory_all_unreachable_cb(evutil_socket_t fd, short event, void *arg) control_event_general_error("DIR_ALL_UNREACHABLE"); } -static struct event *directory_all_unreachable_cb_event = NULL; +static mainloop_event_t *directory_all_unreachable_cb_event = NULL; /** We've just tried every dirserver we know about, and none of * them were reachable. Assume the network is down. Change state @@ -1094,16 +1113,15 @@ directory_all_unreachable(time_t now) { (void)now; - stats_n_seconds_working=0; /* reset it */ + reset_uptime(); /* reset it */ if (!directory_all_unreachable_cb_event) { directory_all_unreachable_cb_event = - tor_event_new(tor_libevent_get_base(), - -1, EV_READ, directory_all_unreachable_cb, NULL); + mainloop_event_new(directory_all_unreachable_cb, NULL); tor_assert(directory_all_unreachable_cb_event); } - event_active(directory_all_unreachable_cb_event, EV_READ, 1); + mainloop_event_activate(directory_all_unreachable_cb_event); } /** This function is called whenever we successfully pull down some new @@ -1143,7 +1161,7 @@ directory_info_has_arrived(time_t now, int from_cache, int suppress_logs) if (server_mode(options) && !net_is_disabled() && !from_cache && (have_completed_a_circuit() || !any_predicted_circuits(now))) - consider_testing_reachability(1, 1); + router_do_reachability_checks(1, 1); } /** Perform regular maintenance tasks for a single connection. This @@ -1159,7 +1177,7 @@ run_connection_housekeeping(int i, time_t now) channel_t *chan = NULL; int have_any_circuits; int past_keepalive = - now >= conn->timestamp_lastwritten + options->KeepalivePeriod; + now >= conn->timestamp_last_write_allowed + options->KeepalivePeriod; if (conn->outbuf && !connection_get_outbuf_len(conn) && conn->type == CONN_TYPE_OR) @@ -1174,10 +1192,10 @@ run_connection_housekeeping(int i, time_t now) * if a server or received if a client) for 5 min */ if (conn->type == CONN_TYPE_DIR && ((DIR_CONN_IS_SERVER(conn) && - conn->timestamp_lastwritten + conn->timestamp_last_write_allowed + options->TestingDirConnectionMaxStall < now) || (!DIR_CONN_IS_SERVER(conn) && - conn->timestamp_lastread + conn->timestamp_last_read_allowed + options->TestingDirConnectionMaxStall < now))) { log_info(LD_DIR,"Expiring wedged directory conn (fd %d, purpose %d)", (int)conn->s, conn->purpose); @@ -1235,7 +1253,8 @@ run_connection_housekeeping(int i, time_t now) } else if (we_are_hibernating() && ! have_any_circuits && !connection_get_outbuf_len(conn)) { - /* We're hibernating, there's no circuits, and nothing to flush.*/ + /* We're hibernating or shutting down, there's no circuits, and nothing to + * flush.*/ log_info(LD_OR,"Expiring non-used OR connection to fd %d (%s:%d) " "[Hibernating or exiting].", (int)conn->s,conn->address, conn->port); @@ -1253,13 +1272,14 @@ run_connection_housekeeping(int i, time_t now) connection_or_close_normally(TO_OR_CONN(conn), 0); } else if ( now >= or_conn->timestamp_lastempty + options->KeepalivePeriod*10 && - now >= conn->timestamp_lastwritten + options->KeepalivePeriod*10) { + now >= + conn->timestamp_last_write_allowed + options->KeepalivePeriod*10) { log_fn(LOG_PROTOCOL_WARN,LD_PROTOCOL, "Expiring stuck OR connection to fd %d (%s:%d). (%d bytes to " "flush; %d seconds since last write)", (int)conn->s, conn->address, conn->port, (int)connection_get_outbuf_len(conn), - (int)(now-conn->timestamp_lastwritten)); + (int)(now-conn->timestamp_last_write_allowed)); connection_or_close_normally(TO_OR_CONN(conn), 0); } else if (past_keepalive && !connection_get_outbuf_len(conn)) { /* send a padding cell */ @@ -1296,6 +1316,16 @@ signewnym_impl(time_t now) control_event_signal(SIGNEWNYM); } +/** Callback: run a deferred signewnym. */ +static void +handle_deferred_signewnym_cb(mainloop_event_t *event, void *arg) +{ + (void)event; + (void)arg; + log_info(LD_CONTROL, "Honoring delayed NEWNYM request"); + signewnym_impl(time(NULL)); +} + /** Return the number of times that signewnym has been called. */ unsigned get_signewnym_epoch(void) @@ -1311,71 +1341,106 @@ static int periodic_events_initialized = 0; #undef CALLBACK #define CALLBACK(name) \ static int name ## _callback(time_t, const or_options_t *) -CALLBACK(rotate_onion_key); -CALLBACK(check_onion_keys_expiry_time); -CALLBACK(check_ed_keys); -CALLBACK(launch_descriptor_fetches); -CALLBACK(rotate_x509_certificate); CALLBACK(add_entropy); -CALLBACK(launch_reachability_tests); -CALLBACK(downrate_stability); -CALLBACK(save_stability); CALLBACK(check_authority_cert); +CALLBACK(check_canonical_channels); +CALLBACK(check_descriptor); +CALLBACK(check_dns_honesty); +CALLBACK(check_ed_keys); CALLBACK(check_expired_networkstatus); -CALLBACK(write_stats_file); -CALLBACK(record_bridge_stats); +CALLBACK(check_for_reachability_bw); +CALLBACK(check_onion_keys_expiry_time); CALLBACK(clean_caches); +CALLBACK(clean_consdiffmgr); +CALLBACK(dirvote); +CALLBACK(downrate_stability); +CALLBACK(expire_old_ciruits_serverside); +CALLBACK(fetch_networkstatus); +CALLBACK(heartbeat); +CALLBACK(hs_service); +CALLBACK(launch_descriptor_fetches); +CALLBACK(launch_reachability_tests); +CALLBACK(reachability_warnings); +CALLBACK(record_bridge_stats); CALLBACK(rend_cache_failure_clean); +CALLBACK(reset_padding_counts); CALLBACK(retry_dns); -CALLBACK(check_descriptor); -CALLBACK(check_for_reachability_bw); -CALLBACK(fetch_networkstatus); CALLBACK(retry_listeners); -CALLBACK(expire_old_ciruits_serverside); -CALLBACK(check_dns_honesty); +CALLBACK(rotate_onion_key); +CALLBACK(rotate_x509_certificate); +CALLBACK(save_stability); +CALLBACK(save_state); CALLBACK(write_bridge_ns); -CALLBACK(check_fw_helper_app); -CALLBACK(heartbeat); -CALLBACK(clean_consdiffmgr); -CALLBACK(reset_padding_counts); -CALLBACK(check_canonical_channels); -CALLBACK(hs_service); +CALLBACK(write_stats_file); #undef CALLBACK /* Now we declare an array of periodic_event_item_t for each periodic event */ -#define CALLBACK(name) PERIODIC_EVENT(name) - -static periodic_event_item_t periodic_events[] = { - CALLBACK(rotate_onion_key), - CALLBACK(check_onion_keys_expiry_time), - CALLBACK(check_ed_keys), - CALLBACK(launch_descriptor_fetches), - CALLBACK(rotate_x509_certificate), - CALLBACK(add_entropy), - CALLBACK(launch_reachability_tests), - CALLBACK(downrate_stability), - CALLBACK(save_stability), - CALLBACK(check_authority_cert), - CALLBACK(check_expired_networkstatus), - CALLBACK(write_stats_file), - CALLBACK(record_bridge_stats), - CALLBACK(clean_caches), - CALLBACK(rend_cache_failure_clean), - CALLBACK(retry_dns), - CALLBACK(check_descriptor), - CALLBACK(check_for_reachability_bw), - CALLBACK(fetch_networkstatus), - CALLBACK(retry_listeners), - CALLBACK(expire_old_ciruits_serverside), - CALLBACK(check_dns_honesty), - CALLBACK(write_bridge_ns), - CALLBACK(check_fw_helper_app), - CALLBACK(heartbeat), - CALLBACK(clean_consdiffmgr), - CALLBACK(reset_padding_counts), - CALLBACK(check_canonical_channels), - CALLBACK(hs_service), +#define CALLBACK(name, r, f) PERIODIC_EVENT(name, r, f) + +STATIC periodic_event_item_t periodic_events[] = { + /* Everyone needs to run those. */ + CALLBACK(add_entropy, PERIODIC_EVENT_ROLE_ALL, 0), + CALLBACK(check_expired_networkstatus, PERIODIC_EVENT_ROLE_ALL, 0), + CALLBACK(clean_caches, PERIODIC_EVENT_ROLE_ALL, 0), + CALLBACK(fetch_networkstatus, PERIODIC_EVENT_ROLE_ALL, + PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(heartbeat, PERIODIC_EVENT_ROLE_ALL, 0), + CALLBACK(launch_descriptor_fetches, PERIODIC_EVENT_ROLE_ALL, + PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(reset_padding_counts, PERIODIC_EVENT_ROLE_ALL, 0), + CALLBACK(retry_listeners, PERIODIC_EVENT_ROLE_ALL, + PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(save_state, PERIODIC_EVENT_ROLE_ALL, 0), + CALLBACK(rotate_x509_certificate, PERIODIC_EVENT_ROLE_ALL, 0), + CALLBACK(write_stats_file, PERIODIC_EVENT_ROLE_ALL, 0), + + /* Routers (bridge and relay) only. */ + CALLBACK(check_descriptor, PERIODIC_EVENT_ROLE_ROUTER, + PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(check_ed_keys, PERIODIC_EVENT_ROLE_ROUTER, 0), + CALLBACK(check_for_reachability_bw, PERIODIC_EVENT_ROLE_ROUTER, + PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(check_onion_keys_expiry_time, PERIODIC_EVENT_ROLE_ROUTER, 0), + CALLBACK(expire_old_ciruits_serverside, PERIODIC_EVENT_ROLE_ROUTER, + PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(reachability_warnings, PERIODIC_EVENT_ROLE_ROUTER, + PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(retry_dns, PERIODIC_EVENT_ROLE_ROUTER, 0), + CALLBACK(rotate_onion_key, PERIODIC_EVENT_ROLE_ROUTER, 0), + + /* Authorities (bridge and directory) only. */ + CALLBACK(downrate_stability, PERIODIC_EVENT_ROLE_AUTHORITIES, 0), + CALLBACK(launch_reachability_tests, PERIODIC_EVENT_ROLE_AUTHORITIES, + PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(save_stability, PERIODIC_EVENT_ROLE_AUTHORITIES, 0), + + /* Directory authority only. */ + CALLBACK(check_authority_cert, PERIODIC_EVENT_ROLE_DIRAUTH, 0), + CALLBACK(dirvote, PERIODIC_EVENT_ROLE_DIRAUTH, PERIODIC_EVENT_FLAG_NEED_NET), + + /* Relay only. */ + CALLBACK(check_canonical_channels, PERIODIC_EVENT_ROLE_RELAY, + PERIODIC_EVENT_FLAG_NEED_NET), + CALLBACK(check_dns_honesty, PERIODIC_EVENT_ROLE_RELAY, + PERIODIC_EVENT_FLAG_NEED_NET), + + /* Hidden Service service only. */ + CALLBACK(hs_service, PERIODIC_EVENT_ROLE_HS_SERVICE, + PERIODIC_EVENT_FLAG_NEED_NET), + + /* Bridge only. */ + CALLBACK(record_bridge_stats, PERIODIC_EVENT_ROLE_BRIDGE, 0), + + /* Client only. */ + CALLBACK(rend_cache_failure_clean, PERIODIC_EVENT_ROLE_CLIENT, 0), + + /* Bridge Authority only. */ + CALLBACK(write_bridge_ns, PERIODIC_EVENT_ROLE_BRIDGEAUTH, 0), + + /* Directory server only. */ + CALLBACK(clean_consdiffmgr, PERIODIC_EVENT_ROLE_DIRSERVER, 0), + END_OF_PERIODIC_EVENTS }; #undef CALLBACK @@ -1385,9 +1450,11 @@ static periodic_event_item_t periodic_events[] = { * can access them by name. We also keep them inside periodic_events[] * so that we can implement "reset all timers" in a reasonable way. */ static periodic_event_item_t *check_descriptor_event=NULL; +static periodic_event_item_t *dirvote_event=NULL; static periodic_event_item_t *fetch_networkstatus_event=NULL; static periodic_event_item_t *launch_descriptor_fetches_event=NULL; static periodic_event_item_t *check_dns_honesty_event=NULL; +static periodic_event_item_t *save_state_event=NULL; /** Reset all the periodic events so we'll do all our actions again as if we * just started up. @@ -1417,6 +1484,39 @@ find_periodic_event(const char *name) return NULL; } +/** Return a bitmask of the roles this tor instance is configured for using + * the given options. */ +STATIC int +get_my_roles(const or_options_t *options) +{ + tor_assert(options); + + int roles = 0; + int is_bridge = options->BridgeRelay; + int is_relay = server_mode(options); + int is_dirauth = authdir_mode_v3(options); + int is_bridgeauth = authdir_mode_bridge(options); + int is_hidden_service = !!hs_service_get_num_services() || + !!rend_num_services(); + int is_dirserver = dir_server_mode(options); + /* We also consider tor to have the role of a client if the ControlPort is + * set because a lot of things can be done over the control port which + * requires tor to have basic functionnalities. */ + int is_client = options_any_client_port_set(options) || + options->ControlPort_set || + options->OwningControllerFD >= 0; + + if (is_bridge) roles |= PERIODIC_EVENT_ROLE_BRIDGE; + if (is_client) roles |= PERIODIC_EVENT_ROLE_CLIENT; + if (is_relay) roles |= PERIODIC_EVENT_ROLE_RELAY; + if (is_dirauth) roles |= PERIODIC_EVENT_ROLE_DIRAUTH; + if (is_bridgeauth) roles |= PERIODIC_EVENT_ROLE_BRIDGEAUTH; + if (is_hidden_service) roles |= PERIODIC_EVENT_ROLE_HS_SERVICE; + if (is_dirserver) roles |= PERIODIC_EVENT_ROLE_DIRSERVER; + + return roles; +} + /** Event to run initialize_periodic_events_cb */ static struct event *initialize_periodic_events_event = NULL; @@ -1431,11 +1531,10 @@ initialize_periodic_events_cb(evutil_socket_t fd, short events, void *data) (void) fd; (void) events; (void) data; + tor_event_free(initialize_periodic_events_event); - int i; - for (i = 0; periodic_events[i].name; ++i) { - periodic_event_launch(&periodic_events[i]); - } + + rescan_periodic_events(get_options()); } /** Set up all the members of periodic_events[], and configure them all to be @@ -1446,6 +1545,7 @@ initialize_periodic_events(void) tor_assert(periodic_events_initialized == 0); periodic_events_initialized = 1; + /* Set up all periodic events. We'll launch them by roles. */ int i; for (i = 0; periodic_events[i].name; ++i) { periodic_event_setup(&periodic_events[i]); @@ -1455,9 +1555,11 @@ initialize_periodic_events(void) STMT_BEGIN name ## _event = find_periodic_event( #name ); STMT_END NAMED_CALLBACK(check_descriptor); + NAMED_CALLBACK(dirvote); NAMED_CALLBACK(fetch_networkstatus); NAMED_CALLBACK(launch_descriptor_fetches); NAMED_CALLBACK(check_dns_honesty); + NAMED_CALLBACK(save_state); struct timeval one_second = { 1, 0 }; initialize_periodic_events_event = tor_evtimer_new( @@ -1473,6 +1575,58 @@ teardown_periodic_events(void) for (i = 0; periodic_events[i].name; ++i) { periodic_event_destroy(&periodic_events[i]); } + periodic_events_initialized = 0; +} + +/** Do a pass at all our periodic events, disable those we don't need anymore + * and enable those we need now using the given options. */ +void +rescan_periodic_events(const or_options_t *options) +{ + tor_assert(options); + + /* Avoid scanning the event list if we haven't initialized it yet. This is + * particularly useful for unit tests in order to avoid initializing main + * loop events everytime. */ + if (!periodic_events_initialized) { + return; + } + + int roles = get_my_roles(options); + + for (int i = 0; periodic_events[i].name; ++i) { + periodic_event_item_t *item = &periodic_events[i]; + + /* Handle the event flags. */ + if (net_is_disabled() && + (item->flags & PERIODIC_EVENT_FLAG_NEED_NET)) { + continue; + } + + /* Enable the event if needed. It is safe to enable an event that was + * already enabled. Same goes for disabling it. */ + if (item->roles & roles) { + log_debug(LD_GENERAL, "Launching periodic event %s", item->name); + periodic_event_enable(item); + } else { + log_debug(LD_GENERAL, "Disabling periodic event %s", item->name); + periodic_event_disable(item); + } + } +} + +/* We just got new options globally set, see if we need to enabled or disable + * periodic events. */ +void +periodic_events_on_new_options(const or_options_t *options) +{ + /* Only if we've already initialized the events, rescan the list which will + * enable or disable events depending on our roles. This will be called at + * bootup and we don't want this function to initialize the events because + * they aren't set up at this stage. */ + if (periodic_events_initialized) { + rescan_periodic_events(options); + } } /** @@ -1483,8 +1637,9 @@ teardown_periodic_events(void) void reschedule_descriptor_update_check(void) { - tor_assert(check_descriptor_event); - periodic_event_reschedule(check_descriptor_event); + if (check_descriptor_event) { + periodic_event_reschedule(check_descriptor_event); + } } /** @@ -1501,6 +1656,35 @@ reschedule_directory_downloads(void) periodic_event_reschedule(launch_descriptor_fetches_event); } +/** Mainloop callback: clean up circuits, channels, and connections + * that are pending close. */ +static void +postloop_cleanup_cb(mainloop_event_t *ev, void *arg) +{ + (void)ev; + (void)arg; + circuit_close_all_marked(); + close_closeable_connections(); + channel_run_cleanup(); + channel_listener_run_cleanup(); +} + +/** Event to run postloop_cleanup_cb */ +static mainloop_event_t *postloop_cleanup_ev=NULL; + +/** Schedule a post-loop event to clean up marked channels, connections, and + * circuits. */ +void +mainloop_schedule_postloop_cleanup(void) +{ + if (PREDICT_UNLIKELY(postloop_cleanup_ev == NULL)) { + // (It's possible that we can get here if we decide to close a connection + // in the earliest stages of our configuration, before we create events.) + return; + } + mainloop_event_activate(postloop_cleanup_ev); +} + #define LONGEST_TIMER_PERIOD (30 * 86400) /** Helper: Return the number of seconds between <b>now</b> and <b>next</b>, * clipped to the range [1 second, LONGEST_TIMER_PERIOD]. */ @@ -1538,17 +1722,6 @@ run_scheduled_events(time_t now) */ consider_hibernation(now); - /* 0b. If we've deferred a signewnym, make sure it gets handled - * eventually. */ - if (signewnym_is_pending && - time_of_last_signewnym + MAX_SIGNEWNYM_RATE <= now) { - log_info(LD_CONTROL, "Honoring delayed NEWNYM request"); - signewnym_impl(now); - } - - /* 0c. If we've deferred log messages for the controller, handle them now */ - flush_pending_log_callbacks(); - /* Maybe enough time elapsed for us to reconsider a circuit. */ circuit_upgrade_circuits_from_guard_wait(); @@ -1562,10 +1735,6 @@ run_scheduled_events(time_t now) accounting_run_housekeeping(now); } - if (authdir_mode_v3(options)) { - dirvote_act(options, now); - } - /* 3a. Every second, we examine pending circuits and prune the * ones which have been pending for more than a few seconds. * We do this before step 4, so it can try building more if @@ -1599,12 +1768,6 @@ run_scheduled_events(time_t now) circuit_expire_old_circs_as_needed(now); } - if (!net_is_disabled()) { - /* This is usually redundant with circuit_build_needed_circs() above, - * but it is very fast when there is no work to do. */ - connection_ap_attach_pending(0); - } - /* 5. We do housekeeping for each connection... */ channel_update_bad_for_new_circs(NULL, 0); int i; @@ -1612,32 +1775,9 @@ run_scheduled_events(time_t now) run_connection_housekeeping(i, now); } - /* 6. And remove any marked circuits... */ - circuit_close_all_marked(); - - /* 8. and blow away any connections that need to die. have to do this now, - * because if we marked a conn for close and left its socket -1, then - * we'll pass it to poll/select and bad things will happen. - */ - close_closeable_connections(); - - /* 8b. And if anything in our state is ready to get flushed to disk, we - * flush it. */ - or_state_save(now); - - /* 8c. Do channel cleanup just like for connections */ - channel_run_cleanup(); - channel_listener_run_cleanup(); - /* 11b. check pending unconfigured managed proxies */ if (!net_is_disabled() && pt_proxies_configuration_pending()) pt_configure_remaining_proxies(); - - /* 12. launch diff computations. (This is free if there are none to - * launch.) */ - if (dir_server_mode(options)) { - consdiffmgr_rescan(); - } } /* Periodic callback: rotate the onion keys after the period defined by the @@ -1847,6 +1987,40 @@ check_authority_cert_callback(time_t now, const or_options_t *options) } /** + * Scheduled callback: Run directory-authority voting functionality. + * + * The schedule is a bit complicated here, so dirvote_act() manages the + * schedule itself. + **/ +static int +dirvote_callback(time_t now, const or_options_t *options) +{ + if (!authdir_mode_v3(options)) { + tor_assert_nonfatal_unreached(); + return 3600; + } + + time_t next = dirvote_act(options, now); + if (BUG(next == TIME_MAX)) { + /* This shouldn't be returned unless we called dirvote_act() without + * being an authority. If it happens, maybe our configuration will + * fix itself in an hour or so? */ + return 3600; + } + return safe_timer_diff(now, next); +} + +/** Reschedule the directory-authority voting event. Run this whenever the + * schedule has changed. */ +void +reschedule_dirvote(const or_options_t *options) +{ + if (periodic_events_initialized && authdir_mode_v3(options)) { + periodic_event_reschedule(dirvote_event); + } +} + +/** * Periodic callback: If our consensus is too old, recalculate whether * we can actually use it. */ @@ -1869,6 +2043,34 @@ check_expired_networkstatus_callback(time_t now, const or_options_t *options) } /** + * Scheduled callback: Save the state file to disk if appropriate. + */ +static int +save_state_callback(time_t now, const or_options_t *options) +{ + (void) options; + (void) or_state_save(now); // only saves if appropriate + const time_t next_write = get_or_state()->next_write; + if (next_write == TIME_MAX) { + return 86400; + } + return safe_timer_diff(now, next_write); +} + +/** Reschedule the event for saving the state file. + * + * Run this when the state becomes dirty. */ +void +reschedule_or_state_save(void) +{ + if (save_state_event == NULL) { + /* This can happen early on during startup. */ + return; + } + periodic_event_reschedule(save_state_event); +} + +/** * Periodic callback: Write statistics to disk if appropriate. */ static int @@ -1940,14 +2142,14 @@ reset_padding_counts_callback(time_t now, const or_options_t *options) return REPHIST_CELL_PADDING_COUNTS_INTERVAL; } +static int should_init_bridge_stats = 1; + /** * Periodic callback: Write bridge statistics to disk if appropriate. */ static int record_bridge_stats_callback(time_t now, const or_options_t *options) { - static int should_init_bridge_stats = 1; - /* 1h. Check whether we should write bridge statistics to disk. */ if (should_record_bridge_info(options)) { @@ -2062,8 +2264,8 @@ check_for_reachability_bw_callback(time_t now, const or_options_t *options) if (server_mode(options) && (have_completed_a_circuit() || !any_predicted_circuits(now)) && !net_is_disabled()) { - if (stats_n_seconds_working < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) { - consider_testing_reachability(1, dirport_reachability_count==0); + if (get_uptime() < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) { + router_do_reachability_checks(1, dirport_reachability_count==0); if (++dirport_reachability_count > 5) dirport_reachability_count = 0; return 1; @@ -2141,6 +2343,56 @@ expire_old_ciruits_serverside_callback(time_t now, const or_options_t *options) } /** + * Callback: Send warnings if Tor doesn't find its ports reachable. + */ +static int +reachability_warnings_callback(time_t now, const or_options_t *options) +{ + (void) now; + + if (get_uptime() < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) { + return (int)(TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT - get_uptime()); + } + + if (server_mode(options) && + !net_is_disabled() && + have_completed_a_circuit()) { + /* every 20 minutes, check and complain if necessary */ + const routerinfo_t *me = router_get_my_routerinfo(); + if (me && !check_whether_orport_reachable(options)) { + char *address = tor_dup_ip(me->addr); + log_warn(LD_CONFIG,"Your server (%s:%d) has not managed to confirm that " + "its ORPort is reachable. Relays do not publish descriptors " + "until their ORPort and DirPort are reachable. Please check " + "your firewalls, ports, address, /etc/hosts file, etc.", + address, me->or_port); + control_event_server_status(LOG_WARN, + "REACHABILITY_FAILED ORADDRESS=%s:%d", + address, me->or_port); + tor_free(address); + } + + if (me && !check_whether_dirport_reachable(options)) { + char *address = tor_dup_ip(me->addr); + log_warn(LD_CONFIG, + "Your server (%s:%d) has not managed to confirm that its " + "DirPort is reachable. Relays do not publish descriptors " + "until their ORPort and DirPort are reachable. Please check " + "your firewalls, ports, address, /etc/hosts file, etc.", + address, me->dir_port); + control_event_server_status(LOG_WARN, + "REACHABILITY_FAILED DIRADDRESS=%s:%d", + address, me->dir_port); + tor_free(address); + } + } + + return TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT; +} + +static int dns_honesty_first_time = 1; + +/** * Periodic event: if we're an exit, see if our DNS server is telling us * obvious lies. */ @@ -2155,10 +2407,9 @@ check_dns_honesty_callback(time_t now, const or_options_t *options) router_my_exit_policy_is_reject_star()) return PERIODIC_EVENT_NO_UPDATE; - static int first_time = 1; - if (first_time) { + if (dns_honesty_first_time) { /* Don't launch right when we start */ - first_time = 0; + dns_honesty_first_time = 0; return crypto_rand_int_range(60, 180); } @@ -2182,32 +2433,7 @@ write_bridge_ns_callback(time_t now, const or_options_t *options) return PERIODIC_EVENT_NO_UPDATE; } -/** - * Periodic callback: poke the tor-fw-helper app if we're using one. - */ -static int -check_fw_helper_app_callback(time_t now, const or_options_t *options) -{ - if (net_is_disabled() || - ! server_mode(options) || - ! options->PortForwarding || - options->NoExec) { - return PERIODIC_EVENT_NO_UPDATE; - } - /* 11. check the port forwarding app */ - -#define PORT_FORWARDING_CHECK_INTERVAL 5 - smartlist_t *ports_to_forward = get_list_of_ports_to_forward(); - if (ports_to_forward) { - tor_check_port_forwarding(options->PortForwardingHelper, - ports_to_forward, - now); - - SMARTLIST_FOREACH(ports_to_forward, char *, cp, tor_free(cp)); - smartlist_free(ports_to_forward); - } - return PORT_FORWARDING_CHECK_INTERVAL; -} +static int heartbeat_callback_first_time = 1; /** * Periodic callback: write the heartbeat message in the logs. @@ -2218,16 +2444,14 @@ check_fw_helper_app_callback(time_t now, const or_options_t *options) static int heartbeat_callback(time_t now, const or_options_t *options) { - static int first = 1; - /* Check if heartbeat is disabled */ if (!options->HeartbeatPeriod) { return PERIODIC_EVENT_NO_UPDATE; } /* Skip the first one. */ - if (first) { - first = 0; + if (heartbeat_callback_first_time) { + heartbeat_callback_first_time = 0; return options->HeartbeatPeriod; } @@ -2246,7 +2470,7 @@ static int clean_consdiffmgr_callback(time_t now, const or_options_t *options) { (void)now; - if (server_mode(options)) { + if (dir_server_mode(options)) { consdiffmgr_cleanup(); } return CDM_CLEAN_CALLBACK_INTERVAL; @@ -2277,8 +2501,100 @@ hs_service_callback(time_t now, const or_options_t *options) /** Timer: used to invoke second_elapsed_callback() once per second. */ static periodic_timer_t *second_timer = NULL; -/** Number of libevent errors in the last second: we die if we get too many. */ -static int n_libevent_errors = 0; + +/** + * Enable or disable the per-second timer as appropriate, creating it if + * necessary. + */ +void +reschedule_per_second_timer(void) +{ + struct timeval one_second; + one_second.tv_sec = 1; + one_second.tv_usec = 0; + + if (! second_timer) { + second_timer = periodic_timer_new(tor_libevent_get_base(), + &one_second, + second_elapsed_callback, + NULL); + tor_assert(second_timer); + } + + const bool run_per_second_events = + control_any_per_second_event_enabled() || ! net_is_completely_disabled(); + + if (run_per_second_events) { + periodic_timer_launch(second_timer, &one_second); + } else { + periodic_timer_disable(second_timer); + } +} + +/** Last time that update_current_time was called. */ +static time_t current_second = 0; +/** Last time that update_current_time updated current_second. */ +static monotime_coarse_t current_second_last_changed; + +/** + * Set the current time to "now", which should be the value returned by + * time(). Check for clock jumps and track the total number of seconds we + * have been running. + */ +void +update_current_time(time_t now) +{ + if (PREDICT_LIKELY(now == current_second)) { + /* We call this function a lot. Most frequently, the current second + * will not have changed, so we just return. */ + return; + } + + const time_t seconds_elapsed = current_second ? (now - current_second) : 0; + + /* Check the wall clock against the monotonic clock, so we can + * better tell idleness from clock jumps and/or other shenanigans. */ + monotime_coarse_t last_updated; + memcpy(&last_updated, ¤t_second_last_changed, sizeof(last_updated)); + monotime_coarse_get(¤t_second_last_changed); + + /** How much clock jumping do we tolerate? */ +#define NUM_JUMPED_SECONDS_BEFORE_WARN 100 + + /** How much idleness do we tolerate? */ +#define NUM_IDLE_SECONDS_BEFORE_WARN 3600 + + if (seconds_elapsed < -NUM_JUMPED_SECONDS_BEFORE_WARN) { + // moving back in time is always a bad sign. + circuit_note_clock_jumped(seconds_elapsed, false); + } else if (seconds_elapsed >= NUM_JUMPED_SECONDS_BEFORE_WARN) { + /* Compare the monotonic clock to the result of time(). */ + const int32_t monotime_msec_passed = + monotime_coarse_diff_msec32(&last_updated, + ¤t_second_last_changed); + const int monotime_sec_passed = monotime_msec_passed / 1000; + const int discrepancy = monotime_sec_passed - (int)seconds_elapsed; + /* If the monotonic clock deviates from time(NULL), we have a couple of + * possibilities. On some systems, this means we have been suspended or + * sleeping. Everywhere, it can mean that the wall-clock time has + * been changed -- for example, with settimeofday(). + * + * On the other hand, if the monotonic time matches with the wall-clock + * time, we've probably just been idle for a while, with no events firing. + * we tolerate much more of that. + */ + const bool clock_jumped = abs(discrepancy) > 2; + + if (clock_jumped || seconds_elapsed >= NUM_IDLE_SECONDS_BEFORE_WARN) { + circuit_note_clock_jumped(seconds_elapsed, ! clock_jumped); + } + } else if (seconds_elapsed > 0) { + stats_n_seconds_working += seconds_elapsed; + } + + update_approx_time(now); + current_second = now; +} /** Libevent callback: invoked once every second. */ static void @@ -2287,83 +2603,22 @@ second_elapsed_callback(periodic_timer_t *timer, void *arg) /* XXXX This could be sensibly refactored into multiple callbacks, and we * could use Libevent's timers for this rather than checking the current * time against a bunch of timeouts every second. */ - static time_t current_second = 0; time_t now; - size_t bytes_written; - size_t bytes_read; - int seconds_elapsed; - const or_options_t *options = get_options(); (void)timer; (void)arg; - n_libevent_errors = 0; - - /* log_notice(LD_GENERAL, "Tick."); */ now = time(NULL); - update_approx_time(now); - - /* the second has rolled over. check more stuff. */ - seconds_elapsed = current_second ? (int)(now - current_second) : 0; - bytes_read = (size_t)(stats_n_bytes_read - stats_prev_n_read); - bytes_written = (size_t)(stats_n_bytes_written - stats_prev_n_written); - stats_prev_n_read = stats_n_bytes_read; - stats_prev_n_written = stats_n_bytes_written; - - control_event_bandwidth_used((uint32_t)bytes_read,(uint32_t)bytes_written); - control_event_stream_bandwidth_used(); - control_event_conn_bandwidth_used(); - control_event_circ_bandwidth_used(); - control_event_circuit_cell_stats(); - - if (server_mode(options) && - !net_is_disabled() && - seconds_elapsed > 0 && - have_completed_a_circuit() && - stats_n_seconds_working / TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT != - (stats_n_seconds_working+seconds_elapsed) / - TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) { - /* every 20 minutes, check and complain if necessary */ - const routerinfo_t *me = router_get_my_routerinfo(); - if (me && !check_whether_orport_reachable(options)) { - char *address = tor_dup_ip(me->addr); - log_warn(LD_CONFIG,"Your server (%s:%d) has not managed to confirm that " - "its ORPort is reachable. Relays do not publish descriptors " - "until their ORPort and DirPort are reachable. Please check " - "your firewalls, ports, address, /etc/hosts file, etc.", - address, me->or_port); - control_event_server_status(LOG_WARN, - "REACHABILITY_FAILED ORADDRESS=%s:%d", - address, me->or_port); - tor_free(address); - } - if (me && !check_whether_dirport_reachable(options)) { - char *address = tor_dup_ip(me->addr); - log_warn(LD_CONFIG, - "Your server (%s:%d) has not managed to confirm that its " - "DirPort is reachable. Relays do not publish descriptors " - "until their ORPort and DirPort are reachable. Please check " - "your firewalls, ports, address, /etc/hosts file, etc.", - address, me->dir_port); - control_event_server_status(LOG_WARN, - "REACHABILITY_FAILED DIRADDRESS=%s:%d", - address, me->dir_port); - tor_free(address); - } - } + /* We don't need to do this once-per-second any more: time-updating is + * only in this callback _because it is a callback_. It should be fine + * to disable this callback, and the time will still get updated. + */ + update_current_time(now); -/** If more than this many seconds have elapsed, probably the clock - * jumped: doesn't count. */ -#define NUM_JUMPED_SECONDS_BEFORE_WARN 100 - if (seconds_elapsed < -NUM_JUMPED_SECONDS_BEFORE_WARN || - seconds_elapsed >= NUM_JUMPED_SECONDS_BEFORE_WARN) { - circuit_note_clock_jumped(seconds_elapsed); - } else if (seconds_elapsed > 0) - stats_n_seconds_working += seconds_elapsed; + /* Maybe some controller events are ready to fire */ + control_per_second_events(); run_scheduled_events(now); - - current_second = now; /* remember which second it is, for next time */ } #ifdef HAVE_SYSTEMD_209 @@ -2379,70 +2634,6 @@ systemd_watchdog_callback(periodic_timer_t *timer, void *arg) } #endif /* defined(HAVE_SYSTEMD_209) */ -/** Timer: used to invoke refill_callback(). */ -static periodic_timer_t *refill_timer = NULL; - -/** Libevent callback: invoked periodically to refill token buckets - * and count r/w bytes. */ -static void -refill_callback(periodic_timer_t *timer, void *arg) -{ - static struct timeval current_millisecond; - struct timeval now; - - size_t bytes_written; - size_t bytes_read; - int milliseconds_elapsed = 0; - int seconds_rolled_over = 0; - - const or_options_t *options = get_options(); - - (void)timer; - (void)arg; - - tor_gettimeofday(&now); - - /* If this is our first time, no time has passed. */ - if (current_millisecond.tv_sec) { - long mdiff = tv_mdiff(¤t_millisecond, &now); - if (mdiff > INT_MAX) - mdiff = INT_MAX; - milliseconds_elapsed = (int)mdiff; - seconds_rolled_over = (int)(now.tv_sec - current_millisecond.tv_sec); - } - - bytes_written = stats_prev_global_write_bucket - global_write_bucket; - bytes_read = stats_prev_global_read_bucket - global_read_bucket; - - stats_n_bytes_read += bytes_read; - stats_n_bytes_written += bytes_written; - if (accounting_is_enabled(options) && milliseconds_elapsed >= 0) - accounting_add_bytes(bytes_read, bytes_written, seconds_rolled_over); - - if (milliseconds_elapsed > 0) - connection_bucket_refill(milliseconds_elapsed, (time_t)now.tv_sec); - - stats_prev_global_read_bucket = global_read_bucket; - stats_prev_global_write_bucket = global_write_bucket; - - current_millisecond = now; /* remember what time it is, for next time */ -} - -#ifndef _WIN32 -/** Called when a possibly ignorable libevent error occurs; ensures that we - * don't get into an infinite loop by ignoring too many errors from - * libevent. */ -static int -got_libevent_error(void) -{ - if (++n_libevent_errors > 8) { - log_err(LD_NET, "Too many libevent errors in one second; dying"); - return -1; - } - return 0; -} -#endif /* !defined(_WIN32) */ - #define UPTIME_CUTOFF_FOR_NEW_BANDWIDTH_TEST (6*60*60) /** Called when our IP address seems to have changed. <b>at_interface</b> @@ -2464,9 +2655,9 @@ ip_address_changed(int at_interface) } } else { if (server) { - if (stats_n_seconds_working > UPTIME_CUTOFF_FOR_NEW_BANDWIDTH_TEST) + if (get_uptime() > UPTIME_CUTOFF_FOR_NEW_BANDWIDTH_TEST) reset_bandwidth_test(); - stats_n_seconds_working = 0; + reset_uptime(); router_reset_reachability(); } } @@ -2587,6 +2778,20 @@ do_hup(void) return 0; } +/** Initialize some mainloop_event_t objects that we require. */ +STATIC void +initialize_mainloop_events(void) +{ + if (!schedule_active_linked_connections_event) { + schedule_active_linked_connections_event = + mainloop_event_postloop_new(schedule_active_linked_connections_cb, NULL); + } + if (!postloop_cleanup_ev) { + postloop_cleanup_ev = + mainloop_event_postloop_new(postloop_cleanup_cb, NULL); + } +} + /** Tor main loop. */ int do_main_loop(void) @@ -2600,6 +2805,8 @@ do_main_loop(void) initialize_periodic_events(); } + initialize_mainloop_events(); + /* initialize dns resolve map, spawn workers if needed */ if (dns_init() < 0) { if (get_options()->ServerDNSAllowBrokenConfig) @@ -2626,8 +2833,6 @@ do_main_loop(void) /* Set up our buckets */ connection_bucket_init(); - stats_prev_global_read_bucket = global_read_bucket; - stats_prev_global_write_bucket = global_write_bucket; /* initialize the bootstrap status events to know we're starting up */ control_event_bootstrap(BOOTSTRAP_STATUS_STARTING, 0); @@ -2691,17 +2896,7 @@ do_main_loop(void) } /* set up once-a-second callback. */ - if (! second_timer) { - struct timeval one_second; - one_second.tv_sec = 1; - one_second.tv_usec = 0; - - second_timer = periodic_timer_new(tor_libevent_get_base(), - &one_second, - second_elapsed_callback, - NULL); - tor_assert(second_timer); - } + reschedule_per_second_timer(); #ifdef HAVE_SYSTEMD_209 uint64_t watchdog_delay; @@ -2725,20 +2920,6 @@ do_main_loop(void) } #endif /* defined(HAVE_SYSTEMD_209) */ - if (!refill_timer) { - struct timeval refill_interval; - int msecs = get_options()->TokenBucketRefillInterval; - - refill_interval.tv_sec = msecs/1000; - refill_interval.tv_usec = (msecs%1000)*1000; - - refill_timer = periodic_timer_new(tor_libevent_get_base(), - &refill_interval, - refill_callback, - NULL); - tor_assert(refill_timer); - } - #ifdef HAVE_SYSTEMD { const int r = sd_notify(0, "READY=1"); @@ -2784,6 +2965,11 @@ do_main_loop(void) return run_main_loop_until_done(); } +#ifndef _WIN32 +/** Rate-limiter for EINVAL-type libevent warnings. */ +static ratelim_t libevent_error_ratelim = RATELIM_INIT(10); +#endif + /** * Run the main loop a single time. Return 0 for "exit"; -1 for "exit with * error", and 1 for "run this again." @@ -2804,17 +2990,12 @@ run_main_loop_once(void) errno = 0; #endif - /* All active linked conns should get their read events activated, - * so that libevent knows to run their callbacks. */ - SMARTLIST_FOREACH(active_linked_connection_lst, connection_t *, conn, - event_active(conn->read_event, EV_READ, 1)); - if (get_options()->MainloopStats) { /* We always enforce that EVLOOP_ONCE is passed to event_base_loop() if we * are collecting main loop statistics. */ called_loop_once = 1; } else { - called_loop_once = smartlist_len(active_linked_connection_lst) ? 1 : 0; + called_loop_once = 0; } /* Make sure we know (about) what time it is. */ @@ -2824,8 +3005,8 @@ run_main_loop_once(void) * an event, or the second ends, or until we have some active linked * connections to trigger events for. Libevent will wait till one * of these happens, then run all the appropriate callbacks. */ - loop_result = event_base_loop(tor_libevent_get_base(), - called_loop_once ? EVLOOP_ONCE : 0); + loop_result = tor_libevent_run_event_loop(tor_libevent_get_base(), + called_loop_once); if (get_options()->MainloopStats) { /* Update our main loop counters. */ @@ -2854,9 +3035,12 @@ run_main_loop_once(void) return -1; #ifndef _WIN32 } else if (e == EINVAL) { - log_warn(LD_NET, "EINVAL from libevent: should you upgrade libevent?"); - if (got_libevent_error()) + log_fn_ratelim(&libevent_error_ratelim, LOG_WARN, LD_NET, + "EINVAL from libevent: should you upgrade libevent?"); + if (libevent_error_ratelim.n_calls_since_last_time > 8) { + log_err(LD_NET, "Too many libevent errors, too fast: dying"); return -1; + } #endif /* !defined(_WIN32) */ } else { tor_assert_nonfatal_once(! ERRNO_IS_EINPROGRESS(e)); @@ -2870,19 +3054,6 @@ run_main_loop_once(void) if (main_loop_should_exit) return 0; - /* And here is where we put callbacks that happen "every time the event loop - * runs." They must be very fast, or else the whole Tor process will get - * slowed down. - * - * Note that this gets called once per libevent loop, which will make it - * happen once per group of events that fire, or once per second. */ - - /* If there are any pending client connections, try attaching them to - * circuits (if we can.) This will be pretty fast if nothing new is - * pending. - */ - connection_ap_attach_pending(0); - return 1; } @@ -2915,6 +3086,7 @@ signal_callback(evutil_socket_t fd, short events, void *arg) (void)fd; (void)events; + update_current_time(time(NULL)); process_signal(sig); } @@ -2977,10 +3149,20 @@ process_signal(int sig) case SIGNEWNYM: { time_t now = time(NULL); if (time_of_last_signewnym + MAX_SIGNEWNYM_RATE > now) { - signewnym_is_pending = 1; + const time_t delay_sec = + time_of_last_signewnym + MAX_SIGNEWNYM_RATE - now; + if (! signewnym_is_pending) { + signewnym_is_pending = 1; + if (!handle_deferred_signewnym_ev) { + handle_deferred_signewnym_ev = + mainloop_event_postloop_new(handle_deferred_signewnym_cb, NULL); + } + const struct timeval delay_tv = { delay_sec, 0 }; + mainloop_event_schedule(handle_deferred_signewnym_ev, &delay_tv); + } log_notice(LD_CONTROL, - "Rate limiting NEWNYM request: delaying by %d second(s)", - (int)(MAX_SIGNEWNYM_RATE+time_of_last_signewnym-now)); + "Rate limiting NEWNYM request: delaying by %d second(s)", + (int)(delay_sec)); } else { signewnym_impl(now); } @@ -3004,6 +3186,13 @@ get_uptime,(void)) return stats_n_seconds_working; } +/** Reset Tor's uptime. */ +MOCK_IMPL(void, +reset_uptime,(void)) +{ + stats_n_seconds_working = 0; +} + /** * Write current memory usage information to the log. */ @@ -3047,13 +3236,13 @@ dumpstats(int severity) i, (int)connection_get_inbuf_len(conn), (int)buf_allocation(conn->inbuf), - (int)(now - conn->timestamp_lastread)); + (int)(now - conn->timestamp_last_read_allowed)); tor_log(severity,LD_GENERAL, "Conn %d: %d bytes waiting on outbuf " "(len %d, last written %d secs ago)",i, (int)connection_get_outbuf_len(conn), (int)buf_allocation(conn->outbuf), - (int)(now - conn->timestamp_lastwritten)); + (int)(now - conn->timestamp_last_write_allowed)); if (conn->type == CONN_TYPE_OR) { or_connection_t *or_conn = TO_OR_CONN(conn); if (or_conn->tls) { @@ -3323,14 +3512,12 @@ tor_init(int argc, char *argv[]) if (strstr(version, "alpha") || strstr(version, "beta")) log_notice(LD_GENERAL, "This version is not a stable Tor release. " "Expect more bugs than usual."); + + tor_compress_log_init_warnings(); } #ifdef HAVE_RUST - char *rust_str = rust_welcome_string(); - if (rust_str != NULL && strlen(rust_str) > 0) { - log_notice(LD_GENERAL, "%s", rust_str); - } - tor_free(rust_str); + rust_log_welcome_string(); #endif /* defined(HAVE_RUST) */ if (network_init()<0) { @@ -3488,6 +3675,9 @@ tor_free_all(int postfork) consdiffmgr_free_all(); hs_free_all(); dos_free_all(); + circuitmux_ewma_free_all(); + accounting_free_all(); + if (!postfork) { config_free_all(); or_state_free_all(); @@ -3508,9 +3698,35 @@ tor_free_all(int postfork) smartlist_free(active_linked_connection_lst); periodic_timer_free(second_timer); teardown_periodic_events(); - periodic_timer_free(refill_timer); tor_event_free(shutdown_did_not_work_event); tor_event_free(initialize_periodic_events_event); + mainloop_event_free(directory_all_unreachable_cb_event); + mainloop_event_free(schedule_active_linked_connections_event); + mainloop_event_free(postloop_cleanup_ev); + mainloop_event_free(handle_deferred_signewnym_ev); + +#ifdef HAVE_SYSTEMD_209 + periodic_timer_free(systemd_watchdog_timer); +#endif + + memset(&global_bucket, 0, sizeof(global_bucket)); + memset(&global_relayed_bucket, 0, sizeof(global_relayed_bucket)); + stats_n_bytes_read = stats_n_bytes_written = 0; + time_of_process_start = 0; + time_of_last_signewnym = 0; + signewnym_is_pending = 0; + newnym_epoch = 0; + called_loop_once = 0; + main_loop_should_exit = 0; + main_loop_exit_value = 0; + can_complete_circuits = 0; + quiet_level = 0; + should_init_bridge_stats = 1; + dns_honesty_first_time = 1; + heartbeat_callback_first_time = 1; + current_second = 0; + memset(¤t_second_last_changed, 0, + sizeof(current_second_last_changed)); if (!postfork) { release_lockfile(); diff --git a/src/or/main.h b/src/or/main.h index c49d216f4e..9dbbc6e5ee 100644 --- a/src/or/main.h +++ b/src/or/main.h @@ -28,6 +28,7 @@ int connection_is_on_closeable_list(connection_t *conn); MOCK_DECL(smartlist_t *, get_connection_array, (void)); MOCK_DECL(uint64_t,get_bytes_read,(void)); MOCK_DECL(uint64_t,get_bytes_written,(void)); +void stats_increment_bytes_read_and_written(uint64_t r, uint64_t w); /** Bitmask for events that we can turn on and off with * connection_watch_events. */ @@ -45,7 +46,6 @@ int connection_is_writing(connection_t *conn); MOCK_DECL(void,connection_stop_writing,(connection_t *conn)); MOCK_DECL(void,connection_start_writing,(connection_t *conn)); -void tell_event_loop_to_run_external_code(void); void tor_shutdown_event_loop_and_exit(int exitcode); int tor_event_loop_shutdown_is_pending(void); @@ -61,8 +61,15 @@ void dns_servers_relaunch_checks(void); void reset_all_main_loop_timers(void); void reschedule_descriptor_update_check(void); void reschedule_directory_downloads(void); +void reschedule_or_state_save(void); +void reschedule_dirvote(const or_options_t *options); +void mainloop_schedule_postloop_cleanup(void); +void rescan_periodic_events(const or_options_t *options); + +void update_current_time(time_t now); MOCK_DECL(long,get_uptime,(void)); +MOCK_DECL(void,reset_uptime,(void)); unsigned get_signewnym_epoch(void); @@ -86,21 +93,27 @@ uint64_t get_main_loop_success_count(void); uint64_t get_main_loop_error_count(void); uint64_t get_main_loop_idle_count(void); +void periodic_events_on_new_options(const or_options_t *options); +void reschedule_per_second_timer(void); + extern time_t time_of_process_start; -extern long stats_n_seconds_working; extern int quiet_level; -extern int global_read_bucket; -extern int global_write_bucket; -extern int global_relayed_read_bucket; -extern int global_relayed_write_bucket; +extern token_bucket_rw_t global_bucket; +extern token_bucket_rw_t global_relayed_bucket; #ifdef MAIN_PRIVATE STATIC void init_connection_lists(void); +STATIC void initialize_mainloop_events(void); STATIC void close_closeable_connections(void); STATIC void initialize_periodic_events(void); STATIC void teardown_periodic_events(void); +STATIC int get_my_roles(const or_options_t *options); #ifdef TOR_UNIT_TESTS extern smartlist_t *connection_array; + +/* We need the periodic_event_item_t definition. */ +#include "periodic.h" +extern periodic_event_item_t periodic_events[]; #endif #endif /* defined(MAIN_PRIVATE) */ diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 040405555c..998eaf74e6 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -48,9 +48,10 @@ #include "connection_or.h" #include "consdiffmgr.h" #include "control.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "directory.h" #include "dirserv.h" -#include "dirvote.h" #include "dos.h" #include "entrynodes.h" #include "hibernate.h" @@ -64,10 +65,14 @@ #include "routerlist.h" #include "routerparse.h" #include "scheduler.h" -#include "shared_random.h" #include "transports.h" #include "torcert.h" #include "channelpadding.h" +#include "voting_schedule.h" + +#include "dirauth/dirvote.h" +#include "dirauth/mode.h" +#include "dirauth/shared_random.h" /** Most recently received and validated v3 "ns"-flavored consensus network * status. */ @@ -237,7 +242,7 @@ router_reload_consensus_networkstatus(void) s = networkstatus_read_cached_consensus_impl(flav, flavor, 1); if (s) { if (networkstatus_set_current_consensus(s, flavor, - flags|NSSET_WAS_WAITING_FOR_CERTS, + flags | NSSET_WAS_WAITING_FOR_CERTS, NULL)) { log_info(LD_FS, "Couldn't load unverified consensus %s networkstatus " "from cache", flavor); @@ -365,9 +370,7 @@ networkstatus_vote_free_(networkstatus_t *ns) digestmap_free(ns->desc_digest_map, NULL); if (ns->sr_info.commits) { - SMARTLIST_FOREACH(ns->sr_info.commits, sr_commit_t *, c, - sr_commit_free(c)); - smartlist_free(ns->sr_info.commits); + dirvote_clear_commits(ns); } tor_free(ns->sr_info.previous_srv); tor_free(ns->sr_info.current_srv); @@ -391,6 +394,20 @@ networkstatus_get_voter_by_id(networkstatus_t *vote, return NULL; } +/** Return the signature made by <b>voter</b> using the algorithm + * <b>alg</b>, or NULL if none is found. */ +document_signature_t * +networkstatus_get_voter_sig_by_alg(const networkstatus_voter_info_t *voter, + digest_algorithm_t alg) +{ + if (!voter->sigs) + return NULL; + SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig, + if (sig->alg == alg) + return sig); + return NULL; +} + /** Check whether the signature <b>sig</b> is correctly signed with the * signing key in <b>cert</b>. Return -1 if <b>cert</b> doesn't match the * signing key; otherwise set the good_signature or bad_signature flag on @@ -951,9 +968,12 @@ update_consensus_networkstatus_downloads(time_t now) continue; } - /* Check if we're waiting for certificates to download */ - if (check_consensus_waiting_for_certs(i, now, &consensus_dl_status[i])) + /** Check if we're waiting for certificates to download. If we are, + * launch download for missing directory authority certificates. */ + if (check_consensus_waiting_for_certs(i, now, &consensus_dl_status[i])) { + update_certificate_downloads(now); continue; + } /* Try the requested attempt */ log_info(LD_DIR, "Launching %s standard networkstatus consensus " @@ -1230,16 +1250,20 @@ should_delay_dir_fetches(const or_options_t *options, const char **msg_out) return 0; } -/** Launch requests for networkstatus documents and authority certificates as - * appropriate. */ +/** Launch requests for networkstatus documents as appropriate. This is called + * when we retry all the connections on a SIGHUP and periodically by a Periodic + * event which checks whether we want to download any networkstatus documents. + */ void update_networkstatus_downloads(time_t now) { const or_options_t *options = get_options(); if (should_delay_dir_fetches(options, NULL)) return; + /** Launch a consensus download request, we will wait for the consensus to + * download and when it completes we will launch a certificate download + * request. */ update_consensus_networkstatus_downloads(now); - update_certificate_downloads(now); } /** Launch requests as appropriate for missing directory authority @@ -1521,7 +1545,7 @@ networkstatus_consensus_has_ipv6(const or_options_t* options) return cons->consensus_method >= MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS; } else { - return cons->consensus_method >= MIN_METHOD_FOR_A_LINES; + return 1; } } @@ -1667,23 +1691,6 @@ networkstatus_set_current_consensus_from_ns(networkstatus_t *c, #endif /* defined(TOR_UNIT_TESTS) */ /** - * Return true if any option is set in <b>options</b> to make us behave - * as a client. - * - * XXXX If we need this elsewhere at any point, we should make it nonstatic - * XXXX and move it into another file. - */ -static int -any_client_port_set(const or_options_t *options) -{ - return (options->SocksPort_set || - options->TransPort_set || - options->NATDPort_set || - options->DNSPort_set || - options->HTTPTunnelPort_set); -} - -/** * Helper for handle_missing_protocol_warning: handles either the * client case (if <b>is_client</b> is set) or the server case otherwise. */ @@ -1718,7 +1725,7 @@ handle_missing_protocol_warning(const networkstatus_t *c, const or_options_t *options) { const int is_server = server_mode(options); - const int is_client = any_client_port_set(options) || !is_server; + const int is_client = options_any_client_port_set(options) || !is_server; if (is_server) handle_missing_protocol_warning_impl(c, 0); @@ -1726,6 +1733,57 @@ handle_missing_protocol_warning(const networkstatus_t *c, handle_missing_protocol_warning_impl(c, 1); } +/** + * Check whether we received a consensus that appears to be coming + * from the future. Because we implicitly trust the directory + * authorities' idea of the current time, we produce a warning if we + * get an early consensus. + * + * If we got a consensus that is time stamped far in the past, that + * could simply have come from a stale cache. Possible ways to get a + * consensus from the future can include: + * + * - enough directory authorities have wrong clocks + * - directory authorities collude to produce misleading time stamps + * - our own clock is wrong (this is by far the most likely) + * + * We neglect highly improbable scenarios that involve actual time + * travel. + */ +STATIC void +warn_early_consensus(const networkstatus_t *c, const char *flavor, + time_t now) +{ + char tbuf[ISO_TIME_LEN+1]; + char dbuf[64]; + long delta = now - c->valid_after; + char *flavormsg = NULL; + +/** If a consensus appears more than this many seconds before it could + * possibly be a sufficiently-signed consensus, declare that our clock + * is skewed. */ +#define EARLY_CONSENSUS_NOTICE_SKEW 60 + + /* We assume that if a majority of dirauths have accurate clocks, + * the earliest that a dirauth with a skewed clock could possibly + * publish a sufficiently-signed consensus is (valid_after - + * dist_seconds). Before that time, the skewed dirauth would be + * unable to obtain enough authority signatures for the consensus to + * be valid. */ + if (now >= c->valid_after - c->dist_seconds - EARLY_CONSENSUS_NOTICE_SKEW) + return; + + format_iso_time(tbuf, c->valid_after); + format_time_interval(dbuf, sizeof(dbuf), delta); + log_warn(LD_GENERAL, "Our clock is %s behind the time published in the " + "consensus network status document (%s UTC). Tor needs an " + "accurate clock to work correctly. Please check your time and " + "date settings!", dbuf, tbuf); + tor_asprintf(&flavormsg, "%s flavor consensus", flavor); + clock_skew_warning(NULL, delta, 1, LD_GENERAL, flavormsg, "CONSENSUS"); + tor_free(flavormsg); +} + /** Try to replace the current cached v3 networkstatus with the one in * <b>consensus</b>. If we don't have enough certificates to validate it, * store it in consensus_waiting_for_certs and launch a certificate fetch. @@ -1768,7 +1826,6 @@ networkstatus_set_current_consensus(const char *consensus, consensus_waiting_for_certs_t *waiting = NULL; time_t current_valid_after = 0; int free_consensus = 1; /* Free 'c' at the end of the function */ - int old_ewma_enabled; int checked_protocols_already = 0; if (flav < 0) { @@ -1834,17 +1891,9 @@ networkstatus_set_current_consensus(const char *consensus, current_valid_after = current_md_consensus->valid_after; } } else { - cached_dir_t *cur; - char buf[128]; - tor_snprintf(buf, sizeof(buf), "cached-%s-consensus", flavor); - consensus_fname = get_cachedir_fname(buf); - tor_snprintf(buf, sizeof(buf), "unverified-%s-consensus", flavor); - unverified_fname = get_cachedir_fname(buf); - cur = dirserv_get_consensus(flavor); - if (cur) { - current_digests = &cur->digests; - current_valid_after = cur->published; - } + tor_assert_nonfatal_unreached(); + result = -2; + goto done; } if (current_digests && @@ -1917,6 +1966,15 @@ networkstatus_set_current_consensus(const char *consensus, } } + /* Signatures from the consensus are verified */ + if (from_cache && was_waiting_for_certs) { + /* We check if the consensus is loaded from disk cache and that it + * it is an unverified consensus. If it is unverified, rename it to + * cached-*-consensus since it has been verified. */ + log_info(LD_DIR, "Unverified consensus signatures verified."); + tor_rename(unverified_fname, consensus_fname); + } + if (!from_cache && flav == usable_consensus_flavor()) control_event_client_status(LOG_NOTICE, "CONSENSUS_ARRIVED"); @@ -1985,24 +2043,16 @@ networkstatus_set_current_consensus(const char *consensus, * the first thing we need to do is recalculate the voting schedule static * object so we can use the timings in there needed by some subsystems * such as hidden service and shared random. */ - dirvote_recalculate_timing(options, now); + voting_schedule_recalculate_timing(options, now); + reschedule_dirvote(options); nodelist_set_consensus(c); /* XXXXNM Microdescs: needs a non-ns variant. ???? NM*/ update_consensus_networkstatus_fetch_time(now); - /* Update ewma and adjust policy if needed; first cache the old value */ - old_ewma_enabled = cell_ewma_enabled(); /* Change the cell EWMA settings */ - cell_ewma_set_scale_factor(options, c); - /* If we just enabled ewma, set the cmux policy on all active channels */ - if (cell_ewma_enabled() && !old_ewma_enabled) { - channel_set_cmux_policy_everywhere(&ewma_policy); - } else if (!cell_ewma_enabled() && old_ewma_enabled) { - /* Turn it off everywhere */ - channel_set_cmux_policy_everywhere(NULL); - } + cmux_ewma_set_options(options, c); /* XXXX this call might be unnecessary here: can changing the * current consensus really alter our view of any OR's rate limits? */ @@ -2036,25 +2086,7 @@ networkstatus_set_current_consensus(const char *consensus, write_str_to_file(consensus_fname, consensus, 0); } -/** If a consensus appears more than this many seconds before its declared - * valid-after time, declare that our clock is skewed. */ -#define EARLY_CONSENSUS_NOTICE_SKEW 60 - - if (now < c->valid_after - EARLY_CONSENSUS_NOTICE_SKEW) { - char tbuf[ISO_TIME_LEN+1]; - char dbuf[64]; - long delta = now - c->valid_after; - char *flavormsg = NULL; - format_iso_time(tbuf, c->valid_after); - format_time_interval(dbuf, sizeof(dbuf), delta); - log_warn(LD_GENERAL, "Our clock is %s behind the time published in the " - "consensus network status document (%s UTC). Tor needs an " - "accurate clock to work correctly. Please check your time and " - "date settings!", dbuf, tbuf); - tor_asprintf(&flavormsg, "%s flavor consensus", flavor); - clock_skew_warning(NULL, delta, 1, LD_GENERAL, flavormsg, "CONSENSUS"); - tor_free(flavormsg); - } + warn_early_consensus(c, flavor, now); /* We got a new consesus. Reset our md fetch fail cache */ microdesc_reset_outdated_dirservers_list(); @@ -2634,6 +2666,25 @@ networkstatus_check_required_protocols(const networkstatus_t *ns, return 0; } +/** Release all storage held in <b>s</b>. */ +void +ns_detached_signatures_free_(ns_detached_signatures_t *s) +{ + if (!s) + return; + if (s->signatures) { + STRMAP_FOREACH(s->signatures, flavor, smartlist_t *, sigs) { + SMARTLIST_FOREACH(sigs, document_signature_t *, sig, + document_signature_free(sig)); + smartlist_free(sigs); + } STRMAP_FOREACH_END; + strmap_free(s->signatures, NULL); + strmap_free(s->digests, tor_free_); + } + + tor_free(s); +} + /** Free all storage held locally in this module. */ void networkstatus_free_all(void) diff --git a/src/or/networkstatus.h b/src/or/networkstatus.h index 1851a55e82..94f85c3c29 100644 --- a/src/or/networkstatus.h +++ b/src/or/networkstatus.h @@ -24,9 +24,16 @@ void routerstatus_free_(routerstatus_t *rs); void networkstatus_vote_free_(networkstatus_t *ns); #define networkstatus_vote_free(ns) \ FREE_AND_NULL(networkstatus_t, networkstatus_vote_free_, (ns)) +void ns_detached_signatures_free_(ns_detached_signatures_t *s); +#define ns_detached_signatures_free(s) \ + FREE_AND_NULL(ns_detached_signatures_t, ns_detached_signatures_free_, (s)) networkstatus_voter_info_t *networkstatus_get_voter_by_id( networkstatus_t *vote, const char *identity); +document_signature_t *networkstatus_get_voter_sig_by_alg( + const networkstatus_voter_info_t *voter, + digest_algorithm_t alg); + int networkstatus_check_consensus_signature(networkstatus_t *consensus, int warn); int networkstatus_check_document_signature(const networkstatus_t *consensus, @@ -144,6 +151,8 @@ void vote_routerstatus_free_(vote_routerstatus_t *rs); #ifdef TOR_UNIT_TESTS STATIC int networkstatus_set_current_consensus_from_ns(networkstatus_t *c, const char *flavor); +STATIC void warn_early_consensus(const networkstatus_t *c, const char *flavor, + time_t now); extern networkstatus_t *current_ns_consensus; extern networkstatus_t *current_md_consensus; #endif /* defined(TOR_UNIT_TESTS) */ diff --git a/src/or/nodelist.c b/src/or/nodelist.c index 82e0926289..032e8d669f 100644 --- a/src/or/nodelist.c +++ b/src/or/nodelist.c @@ -68,6 +68,8 @@ #include <string.h> +#include "dirauth/mode.h" + static void nodelist_drop_node(node_t *node, int remove_from_ht); #define node_free(val) \ FREE_AND_NULL(node_t, node_free_, (val)) @@ -76,10 +78,17 @@ static void node_free_(node_t *node); /** count_usable_descriptors counts descriptors with these flag(s) */ typedef enum { - /* All descriptors regardless of flags */ - USABLE_DESCRIPTOR_ALL = 0, - /* Only descriptors with the Exit flag */ - USABLE_DESCRIPTOR_EXIT_ONLY = 1 + /* All descriptors regardless of flags or exit policies */ + USABLE_DESCRIPTOR_ALL = 0U, + /* Only count descriptors with an exit policy that allows at least one port + */ + USABLE_DESCRIPTOR_EXIT_POLICY = 1U << 0, + /* Only count descriptors for relays that have the exit flag in the + * consensus */ + USABLE_DESCRIPTOR_EXIT_FLAG = 1U << 1, + /* Only count descriptors for relays that have the policy and the flag */ + USABLE_DESCRIPTOR_EXIT_POLICY_AND_FLAG = (USABLE_DESCRIPTOR_EXIT_POLICY | + USABLE_DESCRIPTOR_EXIT_FLAG) } usable_descriptor_t; static void count_usable_descriptors(int *num_present, int *num_usable, @@ -113,6 +122,11 @@ typedef struct nodelist_t { /* Set of addresses that belong to nodes we believe in. */ address_set_t *node_addrs; + + /* The valid-after time of the last live consensus that initialized the + * nodelist. We use this to detect outdated nodelists that need to be + * rebuilt using a newer consensus. */ + time_t live_consensus_valid_after; } nodelist_t; static inline unsigned int @@ -227,7 +241,6 @@ node_get_or_create(const char *identity_digest) smartlist_add(the_nodelist->nodes, node); node->nodelist_idx = smartlist_len(the_nodelist->nodes) - 1; - node->hsdir_index = tor_malloc_zero(sizeof(hsdir_index_t)); node->country = -1; @@ -379,26 +392,26 @@ node_set_hsdir_index(node_t *node, const networkstatus_t *ns) /* Build the fetch index. */ hs_build_hsdir_index(node_identity_pk, fetch_srv, fetch_tp, - node->hsdir_index->fetch); + node->hsdir_index.fetch); /* If we are in the time segment between SRV#N and TP#N, the fetch index is the same as the first store index */ if (!hs_in_period_between_tp_and_srv(ns, now)) { - memcpy(node->hsdir_index->store_first, node->hsdir_index->fetch, - sizeof(node->hsdir_index->store_first)); + memcpy(node->hsdir_index.store_first, node->hsdir_index.fetch, + sizeof(node->hsdir_index.store_first)); } else { hs_build_hsdir_index(node_identity_pk, store_first_srv, store_first_tp, - node->hsdir_index->store_first); + node->hsdir_index.store_first); } /* If we are in the time segment between TP#N and SRV#N+1, the fetch index is the same as the second store index */ if (hs_in_period_between_tp_and_srv(ns, now)) { - memcpy(node->hsdir_index->store_second, node->hsdir_index->fetch, - sizeof(node->hsdir_index->store_second)); + memcpy(node->hsdir_index.store_second, node->hsdir_index.fetch, + sizeof(node->hsdir_index.store_second)); } else { hs_build_hsdir_index(node_identity_pk, store_second_srv, store_second_tp, - node->hsdir_index->store_second); + node->hsdir_index.store_second); } done: @@ -658,6 +671,12 @@ nodelist_set_consensus(networkstatus_t *ns) } } SMARTLIST_FOREACH_END(node); } + + /* If the consensus is live, note down the consensus valid-after that formed + * the nodelist. */ + if (networkstatus_is_live(ns, approx_time())) { + the_nodelist->live_consensus_valid_after = ns->valid_after; + } } /** Helper: return true iff a node has a usable amount of information*/ @@ -749,7 +768,6 @@ node_free_(node_t *node) if (node->md) node->md->held_by_nodes--; tor_assert(node->nodelist_idx == -1); - tor_free(node->hsdir_index); tor_free(node); } @@ -883,6 +901,25 @@ nodelist_assert_ok(void) digestmap_free(dm, NULL); } +/** Ensure that the nodelist has been created with the most recent consensus. + * If that's not the case, make it so. */ +void +nodelist_ensure_freshness(networkstatus_t *ns) +{ + tor_assert(ns); + + /* We don't even have a nodelist: this is a NOP. */ + if (!the_nodelist) { + return; + } + + if (the_nodelist->live_consensus_valid_after != ns->valid_after) { + log_info(LD_GENERAL, "Nodelist was not fresh: rebuilding. (%d / %d)", + (int) the_nodelist->live_consensus_valid_after, + (int) ns->valid_after); + nodelist_set_consensus(ns); + } +} /** Return a list of a node_t * for every node we know about. The caller * MUST NOT modify the list. (You can set and clear flags in the nodes if * you must, but you must not add or remove nodes.) */ @@ -1116,6 +1153,11 @@ node_supports_v3_rendezvous_point(const node_t *node) { tor_assert(node); + /* We can't use a v3 rendezvous point without the curve25519 onion pk. */ + if (!node_get_curve25519_onion_key(node)) { + return 0; + } + return node_get_protover_summary_flags(node)->supports_v3_rendezvous_point; } @@ -1515,9 +1557,11 @@ node_ipv6_or_preferred(const node_t *node) /* XX/teor - node->ipv6_preferred is set from * fascist_firewall_prefer_ipv6_orport() each time the consensus is loaded. */ + node_get_prim_orport(node, &ipv4_addr); if (!fascist_firewall_use_ipv6(options)) { return 0; - } else if (node->ipv6_preferred || node_get_prim_orport(node, &ipv4_addr)) { + } else if (node->ipv6_preferred || + !tor_addr_port_is_valid_ap(&ipv4_addr, 0)) { return node_has_ipv6_orport(node); } return 0; @@ -1528,14 +1572,12 @@ node_ipv6_or_preferred(const node_t *node) if (r && tor_addr_port_is_valid_ipv4h((r)->addr, (r)->port_field, 0)) { \ tor_addr_from_ipv4h(&(ap_out)->addr, (r)->addr); \ (ap_out)->port = (r)->port_field; \ - return 0; \ } \ STMT_END -/** Copy the primary (IPv4) OR port (IP address and TCP port) for - * <b>node</b> into *<b>ap_out</b>. Return 0 if a valid address and - * port was copied, else return non-zero.*/ -int +/** Copy the primary (IPv4) OR port (IP address and TCP port) for <b>node</b> + * into *<b>ap_out</b>. */ +void node_get_prim_orport(const node_t *node, tor_addr_port_t *ap_out) { node_assert_ok(node); @@ -1552,8 +1594,6 @@ node_get_prim_orport(const node_t *node, tor_addr_port_t *ap_out) RETURN_IPV4_AP(node->ri, or_port, ap_out); RETURN_IPV4_AP(node->rs, or_port, ap_out); /* Microdescriptors only have an IPv6 address */ - - return -1; } /** Copy the preferred OR port (IP address and TCP port) for @@ -1578,6 +1618,7 @@ node_get_pref_ipv6_orport(const node_t *node, tor_addr_port_t *ap_out) { node_assert_ok(node); tor_assert(ap_out); + memset(ap_out, 0, sizeof(*ap_out)); /* Check ri first, because rewrite_node_address_for_bridge() updates * node->ri with the configured bridge address. @@ -1625,32 +1666,35 @@ node_ipv6_dir_preferred(const node_t *node) * so we can't use it to determine DirPort IPv6 preference. * This means that bridge clients will use IPv4 DirPorts by default. */ + node_get_prim_dirport(node, &ipv4_addr); if (!fascist_firewall_use_ipv6(options)) { return 0; - } else if (node_get_prim_dirport(node, &ipv4_addr) + } else if (!tor_addr_port_is_valid_ap(&ipv4_addr, 0) || fascist_firewall_prefer_ipv6_dirport(get_options())) { return node_has_ipv6_dirport(node); } return 0; } -/** Copy the primary (IPv4) Dir port (IP address and TCP port) for - * <b>node</b> into *<b>ap_out</b>. Return 0 if a valid address and - * port was copied, else return non-zero.*/ -int +/** Copy the primary (IPv4) Dir port (IP address and TCP port) for <b>node</b> + * into *<b>ap_out</b>. */ +void node_get_prim_dirport(const node_t *node, tor_addr_port_t *ap_out) { node_assert_ok(node); tor_assert(ap_out); + /* Clear the address, as a safety precaution if calling functions ignore the + * return value */ + tor_addr_make_null(&ap_out->addr, AF_INET); + ap_out->port = 0; + /* Check ri first, because rewrite_node_address_for_bridge() updates * node->ri with the configured bridge address. */ RETURN_IPV4_AP(node->ri, dir_port, ap_out); RETURN_IPV4_AP(node->rs, dir_port, ap_out); /* Microdescriptors only have an IPv6 address */ - - return -1; } #undef RETURN_IPV4_AP @@ -2107,8 +2151,11 @@ get_dir_info_status_string(void) * *<b>num_present</b>). * * If <b>in_set</b> is non-NULL, only consider those routers in <b>in_set</b>. - * If <b>exit_only</b> is USABLE_DESCRIPTOR_EXIT_ONLY, only consider nodes - * with the Exit flag. + * If <b>exit_only</b> & USABLE_DESCRIPTOR_EXIT_POLICY, only consider nodes + * present if they have an exit policy that accepts at least one port. + * If <b>exit_only</b> & USABLE_DESCRIPTOR_EXIT_FLAG, only consider nodes + * usable if they have the exit flag in the consensus. + * * If *<b>descs_out</b> is present, add a node_t for each usable descriptor * to it. */ @@ -2129,7 +2176,7 @@ count_usable_descriptors(int *num_present, int *num_usable, if (!node) continue; /* This would be a bug: every entry in the consensus is * supposed to have a node. */ - if (exit_only == USABLE_DESCRIPTOR_EXIT_ONLY && ! rs->is_exit) + if ((exit_only & USABLE_DESCRIPTOR_EXIT_FLAG) && ! rs->is_exit) continue; if (in_set && ! routerset_contains_routerstatus(in_set, rs, -1)) continue; @@ -2142,7 +2189,14 @@ count_usable_descriptors(int *num_present, int *num_usable, else present = NULL != router_get_by_descriptor_digest(digest); if (present) { - /* we have the descriptor listed in the consensus. */ + /* Do the policy check last, because it requires a descriptor, + * and is potentially expensive */ + if ((exit_only & USABLE_DESCRIPTOR_EXIT_POLICY) && + node_exit_policy_rejects_all(node)) { + continue; + } + /* we have the descriptor listed in the consensus, and it + * satisfies our exit constraints (if any) */ ++*num_present; } if (descs_out) @@ -2151,10 +2205,17 @@ count_usable_descriptors(int *num_present, int *num_usable, } SMARTLIST_FOREACH_END(rs); - log_debug(LD_DIR, "%d usable, %d present (%s%s).", + log_debug(LD_DIR, "%d usable, %d present (%s%s%s%s%s).", *num_usable, *num_present, md ? "microdesc" : "desc", - exit_only == USABLE_DESCRIPTOR_EXIT_ONLY ? " exits" : "s"); + (exit_only & USABLE_DESCRIPTOR_EXIT_POLICY_AND_FLAG) ? + " exit" : "s", + (exit_only & USABLE_DESCRIPTOR_EXIT_POLICY) ? + " policies" : "" , + (exit_only == USABLE_DESCRIPTOR_EXIT_POLICY_AND_FLAG) ? + " and" : "" , + (exit_only & USABLE_DESCRIPTOR_EXIT_FLAG) ? + " flags" : "" ); } /** Return an estimate of which fraction of usable paths through the Tor @@ -2189,9 +2250,20 @@ compute_frac_paths_available(const networkstatus_t *consensus, count_usable_descriptors(num_present_out, num_usable_out, mid, consensus, now, NULL, USABLE_DESCRIPTOR_ALL); + log_debug(LD_NET, + "%s: %d present, %d usable", + "mid", + np, + nu); + if (options->EntryNodes) { count_usable_descriptors(&np, &nu, guards, consensus, now, options->EntryNodes, USABLE_DESCRIPTOR_ALL); + log_debug(LD_NET, + "%s: %d present, %d usable", + "guard", + np, + nu); } else { SMARTLIST_FOREACH(mid, const node_t *, node, { if (authdir) { @@ -2202,42 +2274,45 @@ compute_frac_paths_available(const networkstatus_t *consensus, smartlist_add(guards, (node_t*)node); } }); + log_debug(LD_NET, + "%s: %d possible", + "guard", + smartlist_len(guards)); } - /* All nodes with exit flag - * If we're in a network with TestingDirAuthVoteExit set, - * this can cause false positives on have_consensus_path, - * incorrectly setting it to CONSENSUS_PATH_EXIT. This is - * an unavoidable feature of forcing authorities to declare - * certain nodes as exits. - */ + /* All nodes with exit policy and flag */ count_usable_descriptors(&np, &nu, exits, consensus, now, - NULL, USABLE_DESCRIPTOR_EXIT_ONLY); + NULL, USABLE_DESCRIPTOR_EXIT_POLICY_AND_FLAG); log_debug(LD_NET, "%s: %d present, %d usable", "exits", np, nu); - /* We need at least 1 exit present in the consensus to consider + /* We need at least 1 exit (flag and policy) in the consensus to consider * building exit paths */ /* Update our understanding of whether the consensus has exits */ consensus_path_type_t old_have_consensus_path = have_consensus_path; - have_consensus_path = ((nu > 0) ? + have_consensus_path = ((np > 0) ? CONSENSUS_PATH_EXIT : CONSENSUS_PATH_INTERNAL); - if (have_consensus_path == CONSENSUS_PATH_INTERNAL - && old_have_consensus_path != have_consensus_path) { - log_notice(LD_NET, - "The current consensus has no exit nodes. " - "Tor can only build internal paths, " - "such as paths to hidden services."); - - /* However, exit nodes can reachability self-test using this consensus, - * join the network, and appear in a later consensus. This will allow - * the network to build exit paths, such as paths for world wide web - * browsing (as distinct from hidden service web browsing). */ + if (old_have_consensus_path != have_consensus_path) { + if (have_consensus_path == CONSENSUS_PATH_INTERNAL) { + log_notice(LD_NET, + "The current consensus has no exit nodes. " + "Tor can only build internal paths, " + "such as paths to onion services."); + + /* However, exit nodes can reachability self-test using this consensus, + * join the network, and appear in a later consensus. This will allow + * the network to build exit paths, such as paths for world wide web + * browsing (as distinct from hidden service web browsing). */ + } else if (old_have_consensus_path == CONSENSUS_PATH_INTERNAL) { + log_notice(LD_NET, + "The current consensus contains exit nodes. " + "Tor can build exit and internal paths."); + } } f_guard = frac_nodes_with_descriptors(guards, WEIGHT_FOR_GUARD); @@ -2259,43 +2334,28 @@ compute_frac_paths_available(const networkstatus_t *consensus, smartlist_t *myexits= smartlist_new(); smartlist_t *myexits_unflagged = smartlist_new(); - /* All nodes with exit flag in ExitNodes option */ + /* All nodes with exit policy and flag in ExitNodes option */ count_usable_descriptors(&np, &nu, myexits, consensus, now, - options->ExitNodes, USABLE_DESCRIPTOR_EXIT_ONLY); + options->ExitNodes, + USABLE_DESCRIPTOR_EXIT_POLICY_AND_FLAG); log_debug(LD_NET, "%s: %d present, %d usable", "myexits", np, nu); - /* Now compute the nodes in the ExitNodes option where which we don't know - * what their exit policy is, or we know it permits something. */ + /* Now compute the nodes in the ExitNodes option where we know their exit + * policy permits something. */ count_usable_descriptors(&np, &nu, myexits_unflagged, consensus, now, - options->ExitNodes, USABLE_DESCRIPTOR_ALL); + options->ExitNodes, + USABLE_DESCRIPTOR_EXIT_POLICY); log_debug(LD_NET, "%s: %d present, %d usable", "myexits_unflagged (initial)", np, nu); - SMARTLIST_FOREACH_BEGIN(myexits_unflagged, const node_t *, node) { - if (node_has_preferred_descriptor(node, 0) && - node_exit_policy_rejects_all(node)) { - SMARTLIST_DEL_CURRENT(myexits_unflagged, node); - /* this node is not actually an exit */ - np--; - /* this node is unusable as an exit */ - nu--; - } - } SMARTLIST_FOREACH_END(node); - - log_debug(LD_NET, - "%s: %d present, %d usable", - "myexits_unflagged (final)", - np, - nu); - f_myexit= frac_nodes_with_descriptors(myexits,WEIGHT_FOR_EXIT); f_myexit_unflagged= frac_nodes_with_descriptors(myexits_unflagged,WEIGHT_FOR_EXIT); @@ -2336,14 +2396,14 @@ compute_frac_paths_available(const networkstatus_t *consensus, tor_asprintf(status_out, "%d%% of guards bw, " "%d%% of midpoint bw, and " - "%d%% of exit bw%s = " + "%d%% of %s = " "%d%% of path bw", (int)(f_guard*100), (int)(f_mid*100), (int)(f_exit*100), (router_have_consensus_path() == CONSENSUS_PATH_EXIT ? - "" : - " (no exits in consensus)"), + "exit bw" : + "end bw (no exits in consensus)"), (int)(f_path*100)); return f_path; diff --git a/src/or/nodelist.h b/src/or/nodelist.h index 00f12ca1e4..dbe9ad18ff 100644 --- a/src/or/nodelist.h +++ b/src/or/nodelist.h @@ -29,6 +29,7 @@ const node_t *node_get_by_hex_id(const char *identity_digest, node_t *nodelist_set_routerinfo(routerinfo_t *ri, routerinfo_t **ri_old_out); node_t *nodelist_add_microdesc(microdesc_t *md); void nodelist_set_consensus(networkstatus_t *ns); +void nodelist_ensure_freshness(networkstatus_t *ns); int nodelist_probably_contains_address(const tor_addr_t *addr); void nodelist_remove_microdesc(const char *identity_digest, microdesc_t *md); @@ -79,11 +80,11 @@ int node_has_ipv6_dirport(const node_t *node); /* Deprecated - use node_ipv6_or_preferred or node_ipv6_dir_preferred */ #define node_ipv6_preferred(node) node_ipv6_or_preferred(node) int node_ipv6_or_preferred(const node_t *node); -int node_get_prim_orport(const node_t *node, tor_addr_port_t *ap_out); +void node_get_prim_orport(const node_t *node, tor_addr_port_t *ap_out); void node_get_pref_orport(const node_t *node, tor_addr_port_t *ap_out); void node_get_pref_ipv6_orport(const node_t *node, tor_addr_port_t *ap_out); int node_ipv6_dir_preferred(const node_t *node); -int node_get_prim_dirport(const node_t *node, tor_addr_port_t *ap_out); +void node_get_prim_dirport(const node_t *node, tor_addr_port_t *ap_out); void node_get_pref_dirport(const node_t *node, tor_addr_port_t *ap_out); void node_get_pref_ipv6_dirport(const node_t *node, tor_addr_port_t *ap_out); int node_has_curve25519_onion_key(const node_t *node); diff --git a/src/or/ntmain.c b/src/or/ntmain.c index ebbe0018bd..e9a299807a 100644 --- a/src/or/ntmain.c +++ b/src/or/ntmain.c @@ -24,8 +24,6 @@ #include "main.h" #include "ntmain.h" -#include <event2/event.h> - #include <windows.h> #define GENSRV_SERVICENAME "tor" #define GENSRV_DISPLAYNAME "Tor Win32 Service" @@ -245,7 +243,8 @@ nt_service_control(DWORD request) log_notice(LD_GENERAL, "Got stop/shutdown request; shutting down cleanly."); service_status.dwCurrentState = SERVICE_STOP_PENDING; - event_base_loopexit(tor_libevent_get_base(), &exit_now); + tor_libevent_exit_loop_after_delay(tor_libevent_get_base(), + &exit_now); return; } service_fns.SetServiceStatus_fn(hStatus, &service_status); diff --git a/src/or/onion.c b/src/or/onion.c index bd80c2f503..829be12bae 100644 --- a/src/or/onion.c +++ b/src/or/onion.c @@ -67,6 +67,7 @@ #include "circuitlist.h" #include "config.h" #include "cpuworker.h" +#include "crypto_util.h" #include "networkstatus.h" #include "onion.h" #include "onion_fast.h" @@ -521,6 +522,11 @@ onion_skin_create(int type, return r; } +/* This is the maximum value for keys_out_len passed to + * onion_skin_server_handshake, plus 16. We can make it bigger if needed: + * It just defines how many bytes to stack-allocate. */ +#define MAX_KEYS_TMP_LEN 128 + /** Perform the second (server-side) step of a circuit-creation handshake of * type <b>type</b>, responding to the client request in <b>onion_skin</b> * using the keys in <b>keys</b>. On success, write our response into @@ -563,20 +569,21 @@ onion_skin_server_handshake(int type, return -1; { size_t keys_tmp_len = keys_out_len + DIGEST_LEN; - uint8_t *keys_tmp = tor_malloc(keys_out_len + DIGEST_LEN); + tor_assert(keys_tmp_len <= MAX_KEYS_TMP_LEN); + uint8_t keys_tmp[MAX_KEYS_TMP_LEN]; if (onion_skin_ntor_server_handshake( onion_skin, keys->curve25519_key_map, keys->junk_keypair, keys->my_identity, reply_out, keys_tmp, keys_tmp_len)<0) { - tor_free(keys_tmp); + /* no need to memwipe here, since the output will never be used */ return -1; } + memcpy(keys_out, keys_tmp, keys_out_len); memcpy(rend_nonce_out, keys_tmp+keys_out_len, DIGEST_LEN); - memwipe(keys_tmp, 0, keys_tmp_len); - tor_free(keys_tmp); + memwipe(keys_tmp, 0, sizeof(keys_tmp)); r = NTOR_REPLY_LEN; } break; diff --git a/src/or/onion_fast.c b/src/or/onion_fast.c index de9103b1f5..9f9b2199d4 100644 --- a/src/or/onion_fast.c +++ b/src/or/onion_fast.c @@ -29,6 +29,8 @@ #include "or.h" #include "onion_fast.h" +#include "crypto_rand.h" +#include "crypto_util.h" /** Release all state held in <b>victim</b>. */ void diff --git a/src/or/onion_ntor.c b/src/or/onion_ntor.c index b167cb61fb..02d43cb722 100644 --- a/src/or/onion_ntor.c +++ b/src/or/onion_ntor.c @@ -22,6 +22,8 @@ #define ONION_NTOR_PRIVATE #include "crypto.h" +#include "crypto_digest.h" +#include "crypto_util.h" #include "onion_ntor.h" #include "torlog.h" #include "util.h" diff --git a/src/or/onion_tap.c b/src/or/onion_tap.c index c71fa236ed..44737034f4 100644 --- a/src/or/onion_tap.c +++ b/src/or/onion_tap.c @@ -29,6 +29,8 @@ #include "or.h" #include "config.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "onion_tap.h" #include "rephist.h" diff --git a/src/or/or.h b/src/or/or.h index 3c0c2ad613..db8f9544fe 100644 --- a/src/or/or.h +++ b/src/or/or.h @@ -80,6 +80,7 @@ #include "crypto_curve25519.h" #include "crypto_ed25519.h" #include "tor_queue.h" +#include "token_bucket.h" #include "util_format.h" #include "hs_circuitmap.h" @@ -727,8 +728,8 @@ typedef enum { /** Catch-all "other" reason for closing origin circuits. */ #define END_CIRC_AT_ORIGIN -1 -/* Reasons why we (or a remote OR) might close a circuit. See tor-spec.txt for - * documentation of these. */ +/* Reasons why we (or a remote OR) might close a circuit. See tor-spec.txt + * section 5.4 for documentation of these. */ #define END_CIRC_REASON_MIN_ 0 #define END_CIRC_REASON_NONE 0 #define END_CIRC_REASON_TORPROTOCOL 1 @@ -893,8 +894,19 @@ rend_data_v2_t *TO_REND_DATA_V2(const rend_data_t *d) struct hs_ident_edge_conn_t; struct hs_ident_dir_conn_t; struct hs_ident_circuit_t; -/* Stub because we can't include hs_common.h. */ -struct hsdir_index_t; + +/* Hidden service directory index used in a node_t which is set once we set + * the consensus. */ +typedef struct hsdir_index_t { + /* HSDir index to use when fetching a descriptor. */ + uint8_t fetch[DIGEST256_LEN]; + + /* HSDir index used by services to store their first and second + * descriptor. The first descriptor is chronologically older than the second + * one and uses older TP and SRV values. */ + uint8_t store_first[DIGEST256_LEN]; + uint8_t store_second[DIGEST256_LEN]; +} hsdir_index_t; /** Time interval for tracking replays of DH public keys received in * INTRODUCE2 cells. Used only to avoid launching multiple @@ -917,6 +929,7 @@ typedef enum { /** Initial value on both sides of a stream transmission window when the * stream is initialized. Measured in cells. */ #define STREAMWINDOW_START 500 +#define STREAMWINDOW_START_MAX 500 /** Amount to increment a stream window when we get a stream SENDME. */ #define STREAMWINDOW_INCREMENT 50 @@ -1369,10 +1382,10 @@ typedef struct connection_t { * connection. */ size_t outbuf_flushlen; /**< How much data should we try to flush from the * outbuf? */ - time_t timestamp_lastread; /**< When was the last time libevent said we could - * read? */ - time_t timestamp_lastwritten; /**< When was the last time libevent said we - * could write? */ + time_t timestamp_last_read_allowed; /**< When was the last time libevent said + * we could read? */ + time_t timestamp_last_write_allowed; /**< When was the last time libevent + * said we could write? */ time_t timestamp_created; /**< When was this connection_t created? */ @@ -1660,20 +1673,8 @@ typedef struct or_connection_t { time_t timestamp_lastempty; /**< When was the outbuf last completely empty?*/ - /* bandwidth* and *_bucket only used by ORs in OPEN state: */ - int bandwidthrate; /**< Bytes/s added to the bucket. (OPEN ORs only.) */ - int bandwidthburst; /**< Max bucket size for this conn. (OPEN ORs only.) */ - int read_bucket; /**< When this hits 0, stop receiving. Every second we - * add 'bandwidthrate' to this, capping it at - * bandwidthburst. (OPEN ORs only) */ - int write_bucket; /**< When this hits 0, stop writing. Like read_bucket. */ - - /** Last emptied read token bucket in msec since midnight; only used if - * TB_EMPTY events are enabled. */ - uint32_t read_emptied_time; - /** Last emptied write token bucket in msec since midnight; only used if - * TB_EMPTY events are enabled. */ - uint32_t write_emptied_time; + token_bucket_rw_t bucket; /**< Used for rate limiting when the connection is + * in state CONN_OPEN. */ /* * Count the number of bytes flushed out on this orconn, and the number of @@ -2351,10 +2352,10 @@ typedef struct routerstatus_t { * If it's a descriptor, we only use the first DIGEST_LEN bytes. */ char descriptor_digest[DIGEST256_LEN]; uint32_t addr; /**< IPv4 address for this router, in host order. */ - uint16_t or_port; /**< OR port for this router. */ + uint16_t or_port; /**< IPv4 OR port for this router. */ uint16_t dir_port; /**< Directory port for this router. */ tor_addr_t ipv6_addr; /**< IPv6 address for this router. */ - uint16_t ipv6_orport; /**<IPV6 OR port for this router. */ + uint16_t ipv6_orport; /**< IPv6 OR port for this router. */ unsigned int is_authority:1; /**< True iff this router is an authority. */ unsigned int is_exit:1; /**< True iff this router is a good exit. */ unsigned int is_stable:1; /**< True iff this router stays up a long time. */ @@ -2572,7 +2573,7 @@ typedef struct node_t { /* Hidden service directory index data. This is used by a service or client * in order to know what's the hs directory index for this node at the time * the consensus is set. */ - struct hsdir_index_t *hsdir_index; + struct hsdir_index_t hsdir_index; } node_t; /** Linked list of microdesc hash lines for a single router in a directory @@ -2907,11 +2908,7 @@ typedef struct { } u; } onion_handshake_state_t; -/** Holds accounting information for a single step in the layered encryption - * performed by a circuit. Used only at the client edge of a circuit. */ -typedef struct crypt_path_t { - uint32_t magic; - +typedef struct relay_crypto_t { /* crypto environments */ /** Encryption key and counter for cells heading towards the OR at this * step. */ @@ -2925,6 +2922,17 @@ typedef struct crypt_path_t { /** Digest state for cells heading away from the OR at this step. */ crypto_digest_t *b_digest; +} relay_crypto_t; + +/** Holds accounting information for a single step in the layered encryption + * performed by a circuit. Used only at the client edge of a circuit. */ +typedef struct crypt_path_t { + uint32_t magic; + + /** Cryptographic state used for encrypting and authenticating relay + * cells to and from this hop. */ + relay_crypto_t crypto; + /** Current state of the handshake as performed with the OR at this * step. */ onion_handshake_state_t handshake_state; @@ -3170,15 +3178,6 @@ typedef struct circuit_t { /** Index in smartlist of all circuits (global_circuitlist). */ int global_circuitlist_idx; - /** Next circuit in the doubly-linked ring of circuits waiting to add - * cells to n_conn. NULL if we have no cells pending, or if we're not - * linked to an OR connection. */ - struct circuit_t *next_active_on_n_chan; - /** Previous circuit in the doubly-linked ring of circuits waiting to add - * cells to n_conn. NULL if we have no cells pending, or if we're not - * linked to an OR connection. */ - struct circuit_t *prev_active_on_n_chan; - /** Various statistics about cells being added to or removed from this * circuit's queues; used only if CELL_STATS events are enabled and * cleared after being sent to control port. */ @@ -3262,16 +3261,36 @@ typedef struct origin_circuit_t { * associated with this circuit. */ edge_connection_t *p_streams; - /** Bytes read from any attached stream since last call to + /** Bytes read on this circuit since last call to * control_event_circ_bandwidth_used(). Only used if we're configured * to emit CIRC_BW events. */ uint32_t n_read_circ_bw; - /** Bytes written to any attached stream since last call to + /** Bytes written to on this circuit since last call to * control_event_circ_bandwidth_used(). Only used if we're configured * to emit CIRC_BW events. */ uint32_t n_written_circ_bw; + /** Total known-valid relay cell bytes since last call to + * control_event_circ_bandwidth_used(). Only used if we're configured + * to emit CIRC_BW events. */ + uint32_t n_delivered_read_circ_bw; + + /** Total written relay cell bytes since last call to + * control_event_circ_bandwidth_used(). Only used if we're configured + * to emit CIRC_BW events. */ + uint32_t n_delivered_written_circ_bw; + + /** Total overhead data in all known-valid relay data cells since last + * call to control_event_circ_bandwidth_used(). Only used if we're + * configured to emit CIRC_BW events. */ + uint32_t n_overhead_read_circ_bw; + + /** Total written overhead data in all relay data cells since last call to + * control_event_circ_bandwidth_used(). Only used if we're configured + * to emit CIRC_BW events. */ + uint32_t n_overhead_written_circ_bw; + /** Build state for this circuit. It includes the intended path * length, the chosen exit router, rendezvous information, etc. */ @@ -3458,14 +3477,6 @@ struct onion_queue_t; typedef struct or_circuit_t { circuit_t base_; - /** Next circuit in the doubly-linked ring of circuits waiting to add - * cells to p_chan. NULL if we have no cells pending, or if we're not - * linked to an OR connection. */ - struct circuit_t *next_active_on_p_chan; - /** Previous circuit in the doubly-linked ring of circuits waiting to add - * cells to p_chan. NULL if we have no cells pending, or if we're not - * linked to an OR connection. */ - struct circuit_t *prev_active_on_p_chan; /** Pointer to an entry on the onion queue, if this circuit is waiting for a * chance to give an onionskin to a cpuworker. Used only in onion.c */ struct onion_queue_t *onionqueue_entry; @@ -3490,21 +3501,10 @@ typedef struct or_circuit_t { /** Linked list of Exit streams associated with this circuit that are * still being resolved. */ edge_connection_t *resolving_streams; - /** The cipher used by intermediate hops for cells heading toward the - * OP. */ - crypto_cipher_t *p_crypto; - /** The cipher used by intermediate hops for cells heading away from - * the OP. */ - crypto_cipher_t *n_crypto; - - /** The integrity-checking digest used by intermediate hops, for - * cells packaged here and heading towards the OP. - */ - crypto_digest_t *p_digest; - /** The integrity-checking digest used by intermediate hops, for - * cells packaged at the OP and arriving here. - */ - crypto_digest_t *n_digest; + + /** Cryptographic state used for encrypting and authenticating relay + * cells to and from this hop. */ + relay_crypto_t crypto; /** Points to spliced circuit if purpose is REND_ESTABLISHED, and circuit * is not marked for close. */ @@ -4180,6 +4180,8 @@ typedef struct { int NumDirectoryGuards; /**< How many dir guards do we try to establish? * If 0, use value from NumEntryGuards. */ + int NumPrimaryGuards; /**< How many primary guards do we want? */ + int RephistTrackTime; /**< How many seconds do we keep rephist info? */ /** Should we always fetch our dir info on the mirror schedule (which * means directly from the authorities) no matter our other config? */ @@ -4223,10 +4225,6 @@ typedef struct { * testing our DNS server. */ int EnforceDistinctSubnets; /**< If true, don't allow multiple routers in the * same network zone in the same circuit. */ - int PortForwarding; /**< If true, use NAT-PMP or UPnP to automatically - * forward the DirPort and ORPort on the NAT device */ - char *PortForwardingHelper; /** < Filename or full path of the port - forwarding helper executable */ int AllowNonRFC953Hostnames; /**< If true, we allow connections to hostnames * with weird characters. */ /** If true, we try resolving hostnames with weird characters. */ @@ -4352,19 +4350,19 @@ typedef struct { /** Schedule for when servers should download things in general. Only * altered on testing networks. */ - smartlist_t *TestingServerDownloadSchedule; + int TestingServerDownloadInitialDelay; /** Schedule for when clients should download things in general. Only * altered on testing networks. */ - smartlist_t *TestingClientDownloadSchedule; + int TestingClientDownloadInitialDelay; /** Schedule for when servers should download consensuses. Only altered * on testing networks. */ - smartlist_t *TestingServerConsensusDownloadSchedule; + int TestingServerConsensusDownloadInitialDelay; /** Schedule for when clients should download consensuses. Only altered * on testing networks. */ - smartlist_t *TestingClientConsensusDownloadSchedule; + int TestingClientConsensusDownloadInitialDelay; /** Schedule for when clients should download consensuses from authorities * if they are bootstrapping (that is, they don't have a usable, reasonably @@ -4374,7 +4372,7 @@ typedef struct { * This schedule is incremented by (potentially concurrent) connection * attempts, unlike other schedules, which are incremented by connection * failures. Only altered on testing networks. */ - smartlist_t *ClientBootstrapConsensusAuthorityDownloadSchedule; + int ClientBootstrapConsensusAuthorityDownloadInitialDelay; /** Schedule for when clients should download consensuses from fallback * directory mirrors if they are bootstrapping (that is, they don't have a @@ -4384,7 +4382,7 @@ typedef struct { * This schedule is incremented by (potentially concurrent) connection * attempts, unlike other schedules, which are incremented by connection * failures. Only altered on testing networks. */ - smartlist_t *ClientBootstrapConsensusFallbackDownloadSchedule; + int ClientBootstrapConsensusFallbackDownloadInitialDelay; /** Schedule for when clients should download consensuses from authorities * if they are bootstrapping (that is, they don't have a usable, reasonably @@ -4394,15 +4392,15 @@ typedef struct { * This schedule is incremented by (potentially concurrent) connection * attempts, unlike other schedules, which are incremented by connection * failures. Only altered on testing networks. */ - smartlist_t *ClientBootstrapConsensusAuthorityOnlyDownloadSchedule; + int ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay; /** Schedule for when clients should download bridge descriptors. Only * altered on testing networks. */ - smartlist_t *TestingBridgeDownloadSchedule; + int TestingBridgeDownloadInitialDelay; /** Schedule for when clients should download bridge descriptors when they * have no running bridges. Only altered on testing networks. */ - smartlist_t *TestingBridgeBootstrapDownloadSchedule; + int TestingBridgeBootstrapDownloadInitialDelay; /** When directory clients have only a few descriptors to request, they * batch them until they have more, or until this amount of time has @@ -4450,9 +4448,6 @@ typedef struct { /** Enable CELL_STATS events. Only altered on testing networks. */ int TestingEnableCellStatsEvent; - /** Enable TB_EMPTY events. Only altered on testing networks. */ - int TestingEnableTbEmptyEvent; - /** If true, and we have GeoIP data, and we're a bridge, keep a per-country * count of how many client addresses have contacted us so that we can help * the bridge authority guess which countries have blocked access to us. */ @@ -4783,15 +4778,6 @@ typedef struct { time_t LastRotatedOnionKey; } or_state_t; -/** Change the next_write time of <b>state</b> to <b>when</b>, unless the - * state is already scheduled to be written to disk earlier than <b>when</b>. - */ -static inline void or_state_mark_dirty(or_state_t *state, time_t when) -{ - if (state->next_write > when) - state->next_write = when; -} - #define MAX_SOCKS_REPLY_LEN 1024 #define MAX_SOCKS_ADDR_LEN 256 #define SOCKS_NO_AUTH 0x00 diff --git a/src/or/parsecommon.c b/src/or/parsecommon.c index e7d01a5029..cd1a0c2521 100644 --- a/src/or/parsecommon.c +++ b/src/or/parsecommon.c @@ -426,7 +426,7 @@ find_by_keyword_(smartlist_t *s, directory_keyword keyword, * NULL if no such keyword is found. */ directory_token_t * -find_opt_by_keyword(smartlist_t *s, directory_keyword keyword) +find_opt_by_keyword(const smartlist_t *s, directory_keyword keyword) { SMARTLIST_FOREACH(s, directory_token_t *, t, if (t->tp == keyword) return t); return NULL; diff --git a/src/or/parsecommon.h b/src/or/parsecommon.h index 903d94478b..d33faf8ec7 100644 --- a/src/or/parsecommon.h +++ b/src/or/parsecommon.h @@ -314,7 +314,7 @@ directory_token_t *find_by_keyword_(smartlist_t *s, #define find_by_keyword(s, keyword) \ find_by_keyword_((s), (keyword), #keyword) -directory_token_t *find_opt_by_keyword(smartlist_t *s, +directory_token_t *find_opt_by_keyword(const smartlist_t *s, directory_keyword keyword); smartlist_t * find_all_by_keyword(const smartlist_t *s, directory_keyword k); diff --git a/src/or/periodic.c b/src/or/periodic.c index 6896b41c86..9470376d06 100644 --- a/src/or/periodic.c +++ b/src/or/periodic.c @@ -14,10 +14,9 @@ #include "or.h" #include "compat_libevent.h" #include "config.h" +#include "main.h" #include "periodic.h" -#include <event2/event.h> - /** We disable any interval greater than this number of seconds, on the * grounds that it is probably an absolute time mistakenly passed in as a * relative time. @@ -34,24 +33,34 @@ periodic_event_set_interval(periodic_event_item_t *event, struct timeval tv; tv.tv_sec = next_interval; tv.tv_usec = 0; - event_add(event->ev, &tv); + mainloop_event_schedule(event->ev, &tv); } /** Wraps dispatches for periodic events, <b>data</b> will be a pointer to the * event that needs to be called */ static void -periodic_event_dispatch(evutil_socket_t fd, short what, void *data) +periodic_event_dispatch(mainloop_event_t *ev, void *data) { - (void)fd; - (void)what; periodic_event_item_t *event = data; + tor_assert(ev == event->ev); + + if (BUG(!periodic_event_is_enabled(event))) { + return; + } time_t now = time(NULL); + update_current_time(now); const or_options_t *options = get_options(); // log_debug(LD_GENERAL, "Dispatching %s", event->name); int r = event->fn(now, options); int next_interval = 0; + if (!periodic_event_is_enabled(event)) { + /* The event got disabled from inside its callback; no need to + * reschedule. */ + return; + } + /* update the last run time if action was taken */ if (r==0) { log_err(LD_BUG, "Invalid return value for periodic event from %s.", @@ -74,14 +83,17 @@ periodic_event_dispatch(evutil_socket_t fd, short what, void *data) // log_debug(LD_GENERAL, "Scheduling %s for %d seconds", event->name, // next_interval); struct timeval tv = { next_interval , 0 }; - event_add(event->ev, &tv); + mainloop_event_schedule(ev, &tv); } /** Schedules <b>event</b> to run as soon as possible from now. */ void periodic_event_reschedule(periodic_event_item_t *event) { - periodic_event_set_interval(event, 1); + /* Don't reschedule a disabled event. */ + if (periodic_event_is_enabled(event)) { + periodic_event_set_interval(event, 1); + } } /** Initializes the libevent backend for a periodic event. */ @@ -93,10 +105,8 @@ periodic_event_setup(periodic_event_item_t *event) tor_assert(0); } - event->ev = tor_event_new(tor_libevent_get_base(), - -1, 0, - periodic_event_dispatch, - event); + event->ev = mainloop_event_new(periodic_event_dispatch, + event); tor_assert(event->ev); } @@ -109,9 +119,15 @@ periodic_event_launch(periodic_event_item_t *event) log_err(LD_BUG, "periodic_event_launch without periodic_event_setup"); tor_assert(0); } + /* Event already enabled? This is a bug */ + if (periodic_event_is_enabled(event)) { + log_err(LD_BUG, "periodic_event_launch on an already enabled event"); + tor_assert(0); + } // Initial dispatch - periodic_event_dispatch(-1, EV_TIMEOUT, event); + event->enabled = 1; + periodic_event_dispatch(event->ev, event); } /** Release all storage associated with <b>event</b> */ @@ -120,7 +136,38 @@ periodic_event_destroy(periodic_event_item_t *event) { if (!event) return; - tor_event_free(event->ev); + mainloop_event_free(event->ev); event->last_action_time = 0; } +/** Enable the given event by setting its "enabled" flag and scheduling it to + * run immediately in the event loop. This can be called for an event that is + * already enabled. */ +void +periodic_event_enable(periodic_event_item_t *event) +{ + tor_assert(event); + /* Safely and silently ignore if this event is already enabled. */ + if (periodic_event_is_enabled(event)) { + return; + } + + tor_assert(event->ev); + event->enabled = 1; + mainloop_event_activate(event->ev); +} + +/** Disable the given event which means the event is destroyed and then the + * event's enabled flag is unset. This can be called for an event that is + * already disabled. */ +void +periodic_event_disable(periodic_event_item_t *event) +{ + tor_assert(event); + /* Safely and silently ignore if this event is already disabled. */ + if (!periodic_event_is_enabled(event)) { + return; + } + mainloop_event_cancel(event->ev); + event->enabled = 0; +} diff --git a/src/or/periodic.h b/src/or/periodic.h index 8baf3994eb..e8208b2475 100644 --- a/src/or/periodic.h +++ b/src/or/periodic.h @@ -6,6 +6,39 @@ #define PERIODIC_EVENT_NO_UPDATE (-1) +/* Tor roles for which a periodic event item is for. An event can be for + * multiple roles, they can be combined. */ +#define PERIODIC_EVENT_ROLE_CLIENT (1U << 0) +#define PERIODIC_EVENT_ROLE_RELAY (1U << 1) +#define PERIODIC_EVENT_ROLE_BRIDGE (1U << 2) +#define PERIODIC_EVENT_ROLE_DIRAUTH (1U << 3) +#define PERIODIC_EVENT_ROLE_BRIDGEAUTH (1U << 4) +#define PERIODIC_EVENT_ROLE_HS_SERVICE (1U << 5) +#define PERIODIC_EVENT_ROLE_DIRSERVER (1U << 6) + +/* Helper macro to make it a bit less annoying to defined groups of roles that + * are often used. */ + +/* Router that is a Bridge or Relay. */ +#define PERIODIC_EVENT_ROLE_ROUTER \ + (PERIODIC_EVENT_ROLE_BRIDGE | PERIODIC_EVENT_ROLE_RELAY) +/* Authorities that is both bridge and directory. */ +#define PERIODIC_EVENT_ROLE_AUTHORITIES \ + (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_DIRAUTH) +/* All roles. */ +#define PERIODIC_EVENT_ROLE_ALL \ + (PERIODIC_EVENT_ROLE_AUTHORITIES | PERIODIC_EVENT_ROLE_CLIENT | \ + PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_ROUTER) + +/* + * Event flags which can change the behavior of an event. + */ + +/* Indicate that the event needs the network meaning that if we are in + * DisableNetwork or hibernation mode, the event won't be enabled. This obey + * the net_is_disabled() check. */ +#define PERIODIC_EVENT_FLAG_NEED_NET (1U << 0) + /** Callback function for a periodic event to take action. The return value * influences the next time the function will get called. Return * PERIODIC_EVENT_NO_UPDATE to not update <b>last_action_time</b> and be polled @@ -14,24 +47,42 @@ typedef int (*periodic_event_helper_t)(time_t now, const or_options_t *options); -struct event; +struct mainloop_event_t; /** A single item for the periodic-events-function table. */ typedef struct periodic_event_item_t { periodic_event_helper_t fn; /**< The function to run the event */ time_t last_action_time; /**< The last time the function did something */ - struct event *ev; /**< Libevent callback we're using to implement this */ + struct mainloop_event_t *ev; /**< Libevent callback we're using to implement + * this */ const char *name; /**< Name of the function -- for debug */ + + /* Bitmask of roles define above for which this event applies. */ + uint32_t roles; + /* Bitmask of flags which can change the behavior of the event. */ + uint32_t flags; + /* Indicate that this event has been enabled that is scheduled. */ + unsigned int enabled : 1; } periodic_event_item_t; /** events will get their interval from first execution */ -#define PERIODIC_EVENT(fn) { fn##_callback, 0, NULL, #fn } -#define END_OF_PERIODIC_EVENTS { NULL, 0, NULL, NULL } +#define PERIODIC_EVENT(fn, r, f) { fn##_callback, 0, NULL, #fn, r, f, 0 } +#define END_OF_PERIODIC_EVENTS { NULL, 0, NULL, NULL, 0, 0, 0 } + +/* Return true iff the given event was setup before thus is enabled to be + * scheduled. */ +static inline int +periodic_event_is_enabled(const periodic_event_item_t *item) +{ + return item->enabled; +} void periodic_event_launch(periodic_event_item_t *event); void periodic_event_setup(periodic_event_item_t *event); void periodic_event_destroy(periodic_event_item_t *event); void periodic_event_reschedule(periodic_event_item_t *event); +void periodic_event_enable(periodic_event_item_t *event); +void periodic_event_disable(periodic_event_item_t *event); #endif /* !defined(TOR_PERIODIC_H) */ diff --git a/src/or/policies.c b/src/or/policies.c index f718ded326..1210ca687d 100644 --- a/src/or/policies.c +++ b/src/or/policies.c @@ -825,9 +825,8 @@ fascist_firewall_choose_address(const tor_addr_port_t *a, * If pref_only, only choose preferred addresses. In either case, choose * a preferred address before an address that's not preferred. * If both addresses could be chosen (they are both preferred or both allowed) - * choose IPv6 if pref_ipv6 is true, otherwise choose IPv4. - * If neither address is chosen, return 0, else return 1. */ -static int + * choose IPv6 if pref_ipv6 is true, otherwise choose IPv4. */ +static void fascist_firewall_choose_address_base(const tor_addr_t *ipv4_addr, uint16_t ipv4_orport, uint16_t ipv4_dirport, @@ -845,6 +844,9 @@ fascist_firewall_choose_address_base(const tor_addr_t *ipv4_addr, tor_assert(ipv6_addr); tor_assert(ap); + tor_addr_make_null(&ap->addr, AF_UNSPEC); + ap->port = 0; + tor_addr_port_t ipv4_ap; tor_addr_copy(&ipv4_ap.addr, ipv4_addr); ipv4_ap.port = (fw_connection == FIREWALL_OR_CONNECTION @@ -865,17 +867,12 @@ fascist_firewall_choose_address_base(const tor_addr_t *ipv4_addr, if (result) { tor_addr_copy(&ap->addr, &result->addr); ap->port = result->port; - return 1; - } else { - tor_addr_make_null(&ap->addr, AF_UNSPEC); - ap->port = 0; - return 0; } } /** Like fascist_firewall_choose_address_base(), but takes a host-order IPv4 * address as the first parameter. */ -static int +static void fascist_firewall_choose_address_ipv4h(uint32_t ipv4h_addr, uint16_t ipv4_orport, uint16_t ipv4_dirport, @@ -889,11 +886,16 @@ fascist_firewall_choose_address_ipv4h(uint32_t ipv4h_addr, { tor_addr_t ipv4_addr; tor_addr_from_ipv4h(&ipv4_addr, ipv4h_addr); - return fascist_firewall_choose_address_base(&ipv4_addr, ipv4_orport, - ipv4_dirport, ipv6_addr, - ipv6_orport, ipv6_dirport, - fw_connection, pref_only, - pref_ipv6, ap); + tor_assert(ap); + + tor_addr_make_null(&ap->addr, AF_UNSPEC); + ap->port = 0; + + fascist_firewall_choose_address_base(&ipv4_addr, ipv4_orport, + ipv4_dirport, ipv6_addr, + ipv6_orport, ipv6_dirport, + fw_connection, pref_only, + pref_ipv6, ap); } /* Some microdescriptor consensus methods have no IPv6 addresses in rs: they @@ -944,23 +946,25 @@ node_awaiting_ipv6(const or_options_t* options, const node_t *node) * This should only happen when there's no valid consensus, and rs doesn't * correspond to a bridge client's bridge. */ -int +void fascist_firewall_choose_address_rs(const routerstatus_t *rs, firewall_connection_t fw_connection, int pref_only, tor_addr_port_t* ap) { + tor_assert(ap); + + tor_addr_make_null(&ap->addr, AF_UNSPEC); + ap->port = 0; + if (!rs) { - return 0; + return; } - tor_assert(ap); - const or_options_t *options = get_options(); const node_t *node = node_get_by_id(rs->identity_digest); if (node && !node_awaiting_ipv6(options, node)) { - return fascist_firewall_choose_address_node(node, fw_connection, pref_only, - ap); + fascist_firewall_choose_address_node(node, fw_connection, pref_only, ap); } else { /* There's no node-specific IPv6 preference, so use the generic IPv6 * preference instead. */ @@ -970,33 +974,31 @@ fascist_firewall_choose_address_rs(const routerstatus_t *rs, /* Assume IPv4 and IPv6 DirPorts are the same. * Assume the IPv6 OR and Dir addresses are the same. */ - return fascist_firewall_choose_address_ipv4h(rs->addr, - rs->or_port, - rs->dir_port, - &rs->ipv6_addr, - rs->ipv6_orport, - rs->dir_port, - fw_connection, - pref_only, - pref_ipv6, - ap); + fascist_firewall_choose_address_ipv4h(rs->addr, rs->or_port, rs->dir_port, + &rs->ipv6_addr, rs->ipv6_orport, + rs->dir_port, fw_connection, + pref_only, pref_ipv6, ap); } } /** Like fascist_firewall_choose_address_base(), but takes <b>node</b>, and * looks up the node's IPv6 preference rather than taking an argument * for pref_ipv6. */ -int +void fascist_firewall_choose_address_node(const node_t *node, firewall_connection_t fw_connection, int pref_only, tor_addr_port_t *ap) { + tor_assert(ap); + + tor_addr_make_null(&ap->addr, AF_UNSPEC); + ap->port = 0; + if (!node) { - return 0; + return; } node_assert_ok(node); - /* Calling fascist_firewall_choose_address_node() when the node is missing * IPv6 information breaks IPv6-only clients. * If the node is a hard-coded fallback directory or authority, call @@ -1006,7 +1008,7 @@ fascist_firewall_choose_address_node(const node_t *node, * descriptor (routerinfo), or is one of our configured bridges before * calling this function. */ if (BUG(node_awaiting_ipv6(get_options(), node))) { - return 0; + return; } const int pref_ipv6_node = (fw_connection == FIREWALL_OR_CONNECTION @@ -1024,27 +1026,27 @@ fascist_firewall_choose_address_node(const node_t *node, node_get_pref_ipv6_dirport(node, &ipv6_dir_ap); /* Assume the IPv6 OR and Dir addresses are the same. */ - return fascist_firewall_choose_address_base(&ipv4_or_ap.addr, - ipv4_or_ap.port, - ipv4_dir_ap.port, - &ipv6_or_ap.addr, - ipv6_or_ap.port, - ipv6_dir_ap.port, - fw_connection, - pref_only, - pref_ipv6_node, - ap); + fascist_firewall_choose_address_base(&ipv4_or_ap.addr, ipv4_or_ap.port, + ipv4_dir_ap.port, &ipv6_or_ap.addr, + ipv6_or_ap.port, ipv6_dir_ap.port, + fw_connection, pref_only, + pref_ipv6_node, ap); } /** Like fascist_firewall_choose_address_rs(), but takes <b>ds</b>. */ -int +void fascist_firewall_choose_address_dir_server(const dir_server_t *ds, firewall_connection_t fw_connection, int pref_only, tor_addr_port_t *ap) { + tor_assert(ap); + + tor_addr_make_null(&ap->addr, AF_UNSPEC); + ap->port = 0; + if (!ds) { - return 0; + return; } /* A dir_server_t always has a fake_status. As long as it has the same @@ -1052,8 +1054,8 @@ fascist_firewall_choose_address_dir_server(const dir_server_t *ds, * (See #17867.) * This function relies on fascist_firewall_choose_address_rs looking up the * node if it can, because that will get the latest info for the relay. */ - return fascist_firewall_choose_address_rs(&ds->fake_status, fw_connection, - pref_only, ap); + fascist_firewall_choose_address_rs(&ds->fake_status, fw_connection, + pref_only, ap); } /** Return 1 if <b>addr</b> is permitted to connect to our dir port, @@ -2997,11 +2999,12 @@ getinfo_helper_policies(control_connection_t *conn, smartlist_free(private_policy_strings); } else if (!strcmp(question, "exit-policy/reject-private/relay")) { const or_options_t *options = get_options(); - const routerinfo_t *me = router_get_my_routerinfo(); + int err = 0; + const routerinfo_t *me = router_get_my_routerinfo_with_err(&err); if (!me) { - *errmsg = "router_get_my_routerinfo returned NULL"; - return -1; + *errmsg = routerinfo_err_to_string(err); + return routerinfo_err_is_transient(err) ? -1 : 0; } if (!options->ExitPolicyRejectPrivate && @@ -3036,11 +3039,17 @@ getinfo_helper_policies(control_connection_t *conn, SMARTLIST_FOREACH(configured_addresses, tor_addr_t *, a, tor_free(a)); smartlist_free(configured_addresses); } else if (!strcmpstart(question, "exit-policy/")) { - const routerinfo_t *me = router_get_my_routerinfo(); - int include_ipv4 = 0; int include_ipv6 = 0; + int err = 0; + const routerinfo_t *me = router_get_my_routerinfo_with_err(&err); + + if (!me) { + *errmsg = routerinfo_err_to_string(err); + return routerinfo_err_is_transient(err) ? -1 : 0; + } + if (!strcmp(question, "exit-policy/ipv4")) { include_ipv4 = 1; } else if (!strcmp(question, "exit-policy/ipv6")) { @@ -3051,13 +3060,10 @@ getinfo_helper_policies(control_connection_t *conn, return 0; /* No such key. */ } - if (!me) { - *errmsg = "router_get_my_routerinfo returned NULL"; - return -1; - } - - *answer = router_dump_exit_policy_to_string(me,include_ipv4,include_ipv6); + *answer = router_dump_exit_policy_to_string(me,include_ipv4, + include_ipv6); } + return 0; } diff --git a/src/or/policies.h b/src/or/policies.h index 35220a812f..4879acdd8d 100644 --- a/src/or/policies.h +++ b/src/or/policies.h @@ -55,13 +55,13 @@ int fascist_firewall_allows_dir_server(const dir_server_t *ds, firewall_connection_t fw_connection, int pref_only); -int fascist_firewall_choose_address_rs(const routerstatus_t *rs, - firewall_connection_t fw_connection, - int pref_only, tor_addr_port_t* ap); -int fascist_firewall_choose_address_node(const node_t *node, - firewall_connection_t fw_connection, - int pref_only, tor_addr_port_t* ap); -int fascist_firewall_choose_address_dir_server(const dir_server_t *ds, +void fascist_firewall_choose_address_rs(const routerstatus_t *rs, + firewall_connection_t fw_connection, + int pref_only, tor_addr_port_t* ap); +void fascist_firewall_choose_address_node(const node_t *node, + firewall_connection_t fw_connection, + int pref_only, tor_addr_port_t* ap); +void fascist_firewall_choose_address_dir_server(const dir_server_t *ds, firewall_connection_t fw_connection, int pref_only, tor_addr_port_t* ap); diff --git a/src/or/proto_socks.c b/src/or/proto_socks.c index 8700fe1269..57a7d1cd64 100644 --- a/src/or/proto_socks.c +++ b/src/or/proto_socks.c @@ -9,6 +9,7 @@ #include "buffers.h" #include "control.h" #include "config.h" +#include "crypto_util.h" #include "ext_orport.h" #include "proto_socks.h" #include "reasons.h" diff --git a/src/or/protover.c b/src/or/protover.c index a63c2eb02d..c973660980 100644 --- a/src/or/protover.c +++ b/src/or/protover.c @@ -781,7 +781,7 @@ protover_all_supported(const char *s, char **missing_out) versions->high = i; } /* If the last one to be unsupported is one less than the current - * one, we're in a continous range, so set the high field. */ + * one, we're in a continuous range, so set the high field. */ if ((versions->high && versions->high == i - 1) || /* Similarly, if the last high wasn't set and we're currently * one higher than the low, add current index as the highest diff --git a/src/or/protover.h b/src/or/protover.h index b94ebab15b..c46a13de66 100644 --- a/src/or/protover.h +++ b/src/or/protover.h @@ -10,7 +10,7 @@ #define TOR_PROTOVER_H #include "container.h" -#include <stdbool.h> + /** The first version of Tor that included "proto" entries in its * descriptors. Authorities should use this to decide whether to * guess proto lines. */ diff --git a/src/or/relay.c b/src/or/relay.c index 4c1a8ed96d..3632678af6 100644 --- a/src/or/relay.c +++ b/src/or/relay.c @@ -61,6 +61,8 @@ #include "connection_edge.h" #include "connection_or.h" #include "control.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "geoip.h" #include "hs_cache.h" #include "main.h" @@ -70,6 +72,7 @@ #include "policies.h" #include "reasons.h" #include "relay.h" +#include "relay_crypto.h" #include "rendcache.h" #include "rendcommon.h" #include "router.h" @@ -82,9 +85,6 @@ static edge_connection_t *relay_lookup_conn(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction, crypt_path_t *layer_hint); -static int connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, - edge_connection_t *conn, - crypt_path_t *layer_hint); static void circuit_consider_sending_sendme(circuit_t *circ, crypt_path_t *layer_hint); static void circuit_resume_edge_reading(circuit_t *circ, @@ -122,77 +122,6 @@ uint64_t stats_n_circ_max_cell_reached = 0; /** Used to tell which stream to read from first on a circuit. */ static tor_weak_rng_t stream_choice_rng = TOR_WEAK_RNG_INIT; -/** Update digest from the payload of cell. Assign integrity part to - * cell. - */ -static void -relay_set_digest(crypto_digest_t *digest, cell_t *cell) -{ - char integrity[4]; - relay_header_t rh; - - crypto_digest_add_bytes(digest, (char*)cell->payload, CELL_PAYLOAD_SIZE); - crypto_digest_get_digest(digest, integrity, 4); -// log_fn(LOG_DEBUG,"Putting digest of %u %u %u %u into relay cell.", -// integrity[0], integrity[1], integrity[2], integrity[3]); - relay_header_unpack(&rh, cell->payload); - memcpy(rh.integrity, integrity, 4); - relay_header_pack(cell->payload, &rh); -} - -/** Does the digest for this circuit indicate that this cell is for us? - * - * Update digest from the payload of cell (with the integrity part set - * to 0). If the integrity part is valid, return 1, else restore digest - * and cell to their original state and return 0. - */ -static int -relay_digest_matches(crypto_digest_t *digest, cell_t *cell) -{ - uint32_t received_integrity, calculated_integrity; - relay_header_t rh; - crypto_digest_t *backup_digest=NULL; - - backup_digest = crypto_digest_dup(digest); - - relay_header_unpack(&rh, cell->payload); - memcpy(&received_integrity, rh.integrity, 4); - memset(rh.integrity, 0, 4); - relay_header_pack(cell->payload, &rh); - -// log_fn(LOG_DEBUG,"Reading digest of %u %u %u %u from relay cell.", -// received_integrity[0], received_integrity[1], -// received_integrity[2], received_integrity[3]); - - crypto_digest_add_bytes(digest, (char*) cell->payload, CELL_PAYLOAD_SIZE); - crypto_digest_get_digest(digest, (char*) &calculated_integrity, 4); - - if (calculated_integrity != received_integrity) { -// log_fn(LOG_INFO,"Recognized=0 but bad digest. Not recognizing."); -// (%d vs %d).", received_integrity, calculated_integrity); - /* restore digest to its old form */ - crypto_digest_assign(digest, backup_digest); - /* restore the relay header */ - memcpy(rh.integrity, &received_integrity, 4); - relay_header_pack(cell->payload, &rh); - crypto_digest_free(backup_digest); - return 0; - } - crypto_digest_free(backup_digest); - return 1; -} - -/** Apply <b>cipher</b> to CELL_PAYLOAD_SIZE bytes of <b>in</b> - * (in place). - * - * Note that we use the same operation for encrypting and for decrypting. - */ -static void -relay_crypt_one_payload(crypto_cipher_t *cipher, uint8_t *in) -{ - crypto_cipher_crypt_inplace(cipher, (char*) in, CELL_PAYLOAD_SIZE); -} - /** * Update channel usage state based on the type of relay cell and * circuit properties. @@ -297,7 +226,8 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, if (circ->marked_for_close) return 0; - if (relay_crypt(circ, cell, cell_direction, &layer_hint, &recognized) < 0) { + if (relay_decrypt_cell(circ, cell, cell_direction, &layer_hint, &recognized) + < 0) { log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "relay crypt failed. Dropping connection."); return -END_CIRC_REASON_INTERNAL; @@ -402,87 +332,6 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ, return 0; } -/** Do the appropriate en/decryptions for <b>cell</b> arriving on - * <b>circ</b> in direction <b>cell_direction</b>. - * - * If cell_direction == CELL_DIRECTION_IN: - * - If we're at the origin (we're the OP), for hops 1..N, - * decrypt cell. If recognized, stop. - * - Else (we're not the OP), encrypt one hop. Cell is not recognized. - * - * If cell_direction == CELL_DIRECTION_OUT: - * - decrypt one hop. Check if recognized. - * - * If cell is recognized, set *recognized to 1, and set - * *layer_hint to the hop that recognized it. - * - * Return -1 to indicate that we should mark the circuit for close, - * else return 0. - */ -int -relay_crypt(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction, - crypt_path_t **layer_hint, char *recognized) -{ - relay_header_t rh; - - tor_assert(circ); - tor_assert(cell); - tor_assert(recognized); - tor_assert(cell_direction == CELL_DIRECTION_IN || - cell_direction == CELL_DIRECTION_OUT); - - if (cell_direction == CELL_DIRECTION_IN) { - if (CIRCUIT_IS_ORIGIN(circ)) { /* We're at the beginning of the circuit. - * We'll want to do layered decrypts. */ - crypt_path_t *thishop, *cpath = TO_ORIGIN_CIRCUIT(circ)->cpath; - thishop = cpath; - if (thishop->state != CPATH_STATE_OPEN) { - log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, - "Relay cell before first created cell? Closing."); - return -1; - } - do { /* Remember: cpath is in forward order, that is, first hop first. */ - tor_assert(thishop); - - /* decrypt one layer */ - relay_crypt_one_payload(thishop->b_crypto, cell->payload); - - relay_header_unpack(&rh, cell->payload); - if (rh.recognized == 0) { - /* it's possibly recognized. have to check digest to be sure. */ - if (relay_digest_matches(thishop->b_digest, cell)) { - *recognized = 1; - *layer_hint = thishop; - return 0; - } - } - - thishop = thishop->next; - } while (thishop != cpath && thishop->state == CPATH_STATE_OPEN); - log_fn(LOG_PROTOCOL_WARN, LD_OR, - "Incoming cell at client not recognized. Closing."); - return -1; - } else { - /* We're in the middle. Encrypt one layer. */ - relay_crypt_one_payload(TO_OR_CIRCUIT(circ)->p_crypto, cell->payload); - } - } else /* cell_direction == CELL_DIRECTION_OUT */ { - /* We're in the middle. Decrypt one layer. */ - - relay_crypt_one_payload(TO_OR_CIRCUIT(circ)->n_crypto, cell->payload); - - relay_header_unpack(&rh, cell->payload); - if (rh.recognized == 0) { - /* it's possibly recognized. have to check digest to be sure. */ - if (relay_digest_matches(TO_OR_CIRCUIT(circ)->n_digest, cell)) { - *recognized = 1; - return 0; - } - } - } - return 0; -} - /** Package a relay cell from an edge: * - Encrypt it to the right layer * - Append it to the appropriate cell_queue on <b>circ</b>. @@ -501,7 +350,6 @@ circuit_package_relay_cell(cell_t *cell, circuit_t *circ, } if (cell_direction == CELL_DIRECTION_OUT) { - crypt_path_t *thishop; /* counter for repeated crypts */ chan = circ->n_chan; if (!chan) { log_warn(LD_BUG,"outgoing relay cell sent from %s:%d has n_chan==NULL." @@ -524,20 +372,14 @@ circuit_package_relay_cell(cell_t *cell, circuit_t *circ, return 0; /* just drop it */ } - relay_set_digest(layer_hint->f_digest, cell); + relay_encrypt_cell_outbound(cell, TO_ORIGIN_CIRCUIT(circ), layer_hint); - thishop = layer_hint; - /* moving from farthest to nearest hop */ - do { - tor_assert(thishop); - log_debug(LD_OR,"encrypting a layer of the relay cell."); - relay_crypt_one_payload(thishop->f_crypto, cell->payload); - - thishop = thishop->prev; - } while (thishop != TO_ORIGIN_CIRCUIT(circ)->cpath->prev); + /* Update circ written totals for control port */ + origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ); + ocirc->n_written_circ_bw = tor_add_u32_nowrap(ocirc->n_written_circ_bw, + CELL_PAYLOAD_SIZE); } else { /* incoming cell */ - or_circuit_t *or_circ; if (CIRCUIT_IS_ORIGIN(circ)) { /* We should never package an _incoming_ cell from the circuit * origin; that means we messed up somewhere. */ @@ -545,11 +387,9 @@ circuit_package_relay_cell(cell_t *cell, circuit_t *circ, assert_circuit_ok(circ); return 0; /* just drop it */ } - or_circ = TO_OR_CIRCUIT(circ); + or_circuit_t *or_circ = TO_OR_CIRCUIT(circ); + relay_encrypt_cell_inbound(cell, or_circ); chan = or_circ->p_chan; - relay_set_digest(or_circ->p_digest, cell); - /* encrypt one layer */ - relay_crypt_one_payload(or_circ->p_crypto, cell->payload); } ++stats_n_relay_cells_relayed; @@ -770,6 +610,10 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ, tor_free(commands); smartlist_free(commands_list); } + + /* Let's assume we're well-behaved: Anything that we decide to send is + * valid, delivered data. */ + circuit_sent_valid_data(origin_circ, rh.length); } if (circuit_package_relay_cell(&cell, circ, cell_direction, cpath_layer, @@ -899,6 +743,9 @@ connection_ap_process_end_not_open( } } + /* This end cell is now valid. */ + circuit_read_valid_data(circ, rh->length); + if (rh->length == 0) { reason = END_STREAM_REASON_MISC; } @@ -1118,7 +965,12 @@ remap_event_helper(entry_connection_t *conn, const tor_addr_t *new_addr) * header has already been parsed into <b>rh</b>. On success, set * <b>addr_out</b> to the address we're connected to, and <b>ttl_out</b> to * the ttl of that address, in seconds, and return 0. On failure, return - * -1. */ + * -1. + * + * Note that the resulting address can be UNSPEC if the connected cell had no + * address (as for a stream to an union service or a tunneled directory + * connection), and that the ttl can be absent (in which case <b>ttl_out</b> + * is set to -1). */ STATIC int connected_cell_parse(const relay_header_t *rh, const cell_t *cell, tor_addr_t *addr_out, int *ttl_out) @@ -1389,6 +1241,12 @@ connection_edge_process_resolved_cell(edge_connection_t *conn, } } + /* This is valid data at this point. Count it */ + if (conn->on_circuit && CIRCUIT_IS_ORIGIN(conn->on_circuit)) { + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(conn->on_circuit), + rh->length); + } + connection_ap_handshake_socks_got_resolved_cell(entry_conn, errcode, resolved_addresses); @@ -1449,7 +1307,7 @@ connection_edge_process_relay_cell_not_open( "after %d seconds.", (unsigned)circ->n_circ_id, rh->stream_id, - (int)(time(NULL) - conn->base_.timestamp_lastread)); + (int)(time(NULL) - conn->base_.timestamp_last_read_allowed)); if (connected_cell_parse(rh, cell, &addr, &ttl) < 0) { log_fn(LOG_PROTOCOL_WARN, LD_APP, "Got a badly formatted connected cell. Closing."); @@ -1458,6 +1316,9 @@ connection_edge_process_relay_cell_not_open( return 0; } if (tor_addr_family(&addr) != AF_UNSPEC) { + /* The family is not UNSPEC: so we were given an address in the + * connected cell. (This is normal, except for BEGINDIR and onion + * service streams.) */ const sa_family_t family = tor_addr_family(&addr); if (tor_addr_is_null(&addr) || (get_options()->ClientDNSRejectInternalAddresses && @@ -1524,6 +1385,9 @@ connection_edge_process_relay_cell_not_open( entry_conn->pending_optimistic_data = NULL; } + /* This is valid data at this point. Count it */ + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh->length); + /* handle anything that might have queued */ if (connection_edge_package_raw_inbuf(conn, 1, NULL) < 0) { /* (We already sent an end cell if possible) */ @@ -1556,7 +1420,7 @@ connection_edge_process_relay_cell_not_open( * * Return -reason if you want to warn and tear down the circuit, else 0. */ -static int +STATIC int connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, edge_connection_t *conn, crypt_path_t *layer_hint) @@ -1656,7 +1520,6 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, circ->dirreq_id = ++next_id; TO_OR_CIRCUIT(circ)->p_chan->dirreq_id = circ->dirreq_id; } - return connection_exit_begin_conn(cell, circ); case RELAY_COMMAND_DATA: ++stats_n_data_cells_received; @@ -1692,6 +1555,10 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, "(relay data) conn deliver_window below 0. Killing."); return -END_CIRC_REASON_TORPROTOCOL; } + /* Total all valid application bytes delivered */ + if (CIRCUIT_IS_ORIGIN(circ) && rh.length > 0) { + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length); + } stats_n_data_bytes_received += rh.length; connection_buf_add((char*)(cell->payload + RELAY_HEADER_SIZE), @@ -1744,6 +1611,11 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, /* only mark it if not already marked. it's possible to * get the 'end' right around when the client hangs up on us. */ connection_mark_and_flush(TO_CONN(conn)); + + /* Total all valid application bytes delivered */ + if (CIRCUIT_IS_ORIGIN(circ)) { + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length); + } } return 0; case RELAY_COMMAND_EXTEND: @@ -1809,6 +1681,10 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, log_info(domain,"circuit_send_next_onion_skin() failed."); return reason; } + /* Total all valid bytes delivered. */ + if (CIRCUIT_IS_ORIGIN(circ)) { + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length); + } return 0; case RELAY_COMMAND_TRUNCATE: if (layer_hint) { @@ -1874,6 +1750,15 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, log_debug(LD_APP,"circ-level sendme at origin, packagewindow %d.", layer_hint->package_window); circuit_resume_edge_reading(circ, layer_hint); + + /* We count circuit-level sendme's as valid delivered data because + * they are rate limited. + */ + if (CIRCUIT_IS_ORIGIN(circ)) { + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), + rh.length); + } + } else { if (circ->package_window + CIRCWINDOW_INCREMENT > CIRCWINDOW_START_MAX) { @@ -1897,6 +1782,27 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, rh.stream_id); return 0; } + + /* Don't allow the other endpoint to request more than our maximim + * (ie initial) stream SENDME window worth of data. Well-behaved + * stock clients will not request more than this max (as per the check + * in the while loop of connection_edge_consider_sending_sendme()). + */ + if (conn->package_window + STREAMWINDOW_INCREMENT > + STREAMWINDOW_START_MAX) { + static struct ratelim_t stream_warn_ratelim = RATELIM_INIT(600); + log_fn_ratelim(&stream_warn_ratelim,LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Unexpected stream sendme cell. Closing circ (window %d).", + conn->package_window); + return -END_CIRC_REASON_TORPROTOCOL; + } + + /* At this point, the stream sendme is valid */ + if (CIRCUIT_IS_ORIGIN(circ)) { + circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), + rh.length); + } + conn->package_window += STREAMWINDOW_INCREMENT; log_debug(domain,"stream-level sendme, packagewindow now %d.", conn->package_window); @@ -2398,13 +2304,6 @@ circuit_consider_sending_sendme(circuit_t *circ, crypt_path_t *layer_hint) } } -#ifdef ACTIVE_CIRCUITS_PARANOIA -#define assert_cmux_ok_paranoid(chan) \ - assert_circuit_mux_okay(chan) -#else -#define assert_cmux_ok_paranoid(chan) -#endif /* defined(ACTIVE_CIRCUITS_PARANOIA) */ - /** The total number of cells we have allocated. */ static size_t total_cells_allocated = 0; @@ -2692,16 +2591,12 @@ update_circuit_on_cmux_(circuit_t *circ, cell_direction_t direction, } tor_assert(circuitmux_attached_circuit_direction(cmux, circ) == direction); - assert_cmux_ok_paranoid(chan); - /* Update the number of cells we have for the circuit mux */ if (direction == CELL_DIRECTION_OUT) { circuitmux_set_num_cells(cmux, circ, circ->n_chan_cells.n); } else { circuitmux_set_num_cells(cmux, circ, or_circ->p_chan_cells.n); } - - assert_cmux_ok_paranoid(chan); } /** Remove all circuits from the cmux on <b>chan</b>. @@ -2846,7 +2741,6 @@ channel_flush_from_first_active_circuit, (channel_t *chan, int max)) } /* If it returns NULL, no cells left to send */ if (!circ) break; - assert_cmux_ok_paranoid(chan); if (circ->n_chan == chan) { queue = &circ->n_chan_cells; @@ -2950,8 +2844,6 @@ channel_flush_from_first_active_circuit, (channel_t *chan, int max)) } /* Okay, we're done sending now */ - assert_cmux_ok_paranoid(chan); - return n_flushed; } @@ -3012,7 +2904,7 @@ relay_consensus_has_changed(const networkstatus_t *ns) /** Add <b>cell</b> to the queue of <b>circ</b> writing to <b>chan</b> * transmitting in <b>direction</b>. * - * The given <b>cell</b> is copied over the circuit queue so the caller must + * The given <b>cell</b> is copied onto the circuit queue so the caller must * cleanup the memory. * * This function is part of the fast path. */ @@ -3167,17 +3059,6 @@ circuit_clear_cell_queue(circuit_t *circ, channel_t *chan) update_circuit_on_cmux(circ, direction); } -/** Fail with an assert if the circuit mux on chan is corrupt - */ -void -assert_circuit_mux_okay(channel_t *chan) -{ - tor_assert(chan); - tor_assert(chan->cmux); - - circuitmux_assert_okay(chan->cmux); -} - /** Return 1 if we shouldn't restart reading on this circuit, even if * we get a SENDME. Else return 0. */ diff --git a/src/or/relay.h b/src/or/relay.h index e96639170c..ce0969b46c 100644 --- a/src/or/relay.h +++ b/src/or/relay.h @@ -78,7 +78,6 @@ void destroy_cell_queue_append(destroy_cell_queue_t *queue, void channel_unlink_all_circuits(channel_t *chan, smartlist_t *detached_out); MOCK_DECL(int, channel_flush_from_first_active_circuit, (channel_t *chan, int max)); -void assert_circuit_mux_okay(channel_t *chan); void update_circuit_on_cmux_(circuit_t *circ, cell_direction_t direction, const char *file, int lineno); #define update_circuit_on_cmux(circ, direction) \ @@ -92,9 +91,6 @@ void circuit_clear_cell_queue(circuit_t *circ, channel_t *chan); void stream_choice_seed_weak_rng(void); -int relay_crypt(circuit_t *circ, cell_t *cell, cell_direction_t cell_direction, - crypt_path_t **layer_hint, char *recognized); - circid_t packed_cell_get_circid(const packed_cell_t *cell, int wide_circ_ids); #ifdef RELAY_PRIVATE @@ -118,6 +114,10 @@ STATIC packed_cell_t *packed_cell_new(void); STATIC packed_cell_t *cell_queue_pop(cell_queue_t *queue); STATIC destroy_cell_t *destroy_cell_queue_pop(destroy_cell_queue_t *queue); STATIC int cell_queues_check_size(void); +STATIC int connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ, + edge_connection_t *conn, + crypt_path_t *layer_hint); + #endif /* defined(RELAY_PRIVATE) */ #endif /* !defined(TOR_RELAY_H) */ diff --git a/src/or/relay_crypto.c b/src/or/relay_crypto.c new file mode 100644 index 0000000000..530c8e5828 --- /dev/null +++ b/src/or/relay_crypto.c @@ -0,0 +1,327 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "or.h" +#include "config.h" +#include "crypto_util.h" +#include "hs_ntor.h" // for HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN +#include "relay.h" +#include "relay_crypto.h" + +/** Update digest from the payload of cell. Assign integrity part to + * cell. + */ +static void +relay_set_digest(crypto_digest_t *digest, cell_t *cell) +{ + char integrity[4]; + relay_header_t rh; + + crypto_digest_add_bytes(digest, (char*)cell->payload, CELL_PAYLOAD_SIZE); + crypto_digest_get_digest(digest, integrity, 4); +// log_fn(LOG_DEBUG,"Putting digest of %u %u %u %u into relay cell.", +// integrity[0], integrity[1], integrity[2], integrity[3]); + relay_header_unpack(&rh, cell->payload); + memcpy(rh.integrity, integrity, 4); + relay_header_pack(cell->payload, &rh); +} + +/** Does the digest for this circuit indicate that this cell is for us? + * + * Update digest from the payload of cell (with the integrity part set + * to 0). If the integrity part is valid, return 1, else restore digest + * and cell to their original state and return 0. + */ +static int +relay_digest_matches(crypto_digest_t *digest, cell_t *cell) +{ + uint32_t received_integrity, calculated_integrity; + relay_header_t rh; + crypto_digest_checkpoint_t backup_digest; + + crypto_digest_checkpoint(&backup_digest, digest); + + relay_header_unpack(&rh, cell->payload); + memcpy(&received_integrity, rh.integrity, 4); + memset(rh.integrity, 0, 4); + relay_header_pack(cell->payload, &rh); + +// log_fn(LOG_DEBUG,"Reading digest of %u %u %u %u from relay cell.", +// received_integrity[0], received_integrity[1], +// received_integrity[2], received_integrity[3]); + + crypto_digest_add_bytes(digest, (char*) cell->payload, CELL_PAYLOAD_SIZE); + crypto_digest_get_digest(digest, (char*) &calculated_integrity, 4); + + int rv = 1; + + if (calculated_integrity != received_integrity) { +// log_fn(LOG_INFO,"Recognized=0 but bad digest. Not recognizing."); +// (%d vs %d).", received_integrity, calculated_integrity); + /* restore digest to its old form */ + crypto_digest_restore(digest, &backup_digest); + /* restore the relay header */ + memcpy(rh.integrity, &received_integrity, 4); + relay_header_pack(cell->payload, &rh); + rv = 0; + } + + memwipe(&backup_digest, 0, sizeof(backup_digest)); + return rv; +} + +/** Apply <b>cipher</b> to CELL_PAYLOAD_SIZE bytes of <b>in</b> + * (in place). + * + * Note that we use the same operation for encrypting and for decrypting. + */ +static void +relay_crypt_one_payload(crypto_cipher_t *cipher, uint8_t *in) +{ + crypto_cipher_crypt_inplace(cipher, (char*) in, CELL_PAYLOAD_SIZE); +} + +/** Do the appropriate en/decryptions for <b>cell</b> arriving on + * <b>circ</b> in direction <b>cell_direction</b>. + * + * If cell_direction == CELL_DIRECTION_IN: + * - If we're at the origin (we're the OP), for hops 1..N, + * decrypt cell. If recognized, stop. + * - Else (we're not the OP), encrypt one hop. Cell is not recognized. + * + * If cell_direction == CELL_DIRECTION_OUT: + * - decrypt one hop. Check if recognized. + * + * If cell is recognized, set *recognized to 1, and set + * *layer_hint to the hop that recognized it. + * + * Return -1 to indicate that we should mark the circuit for close, + * else return 0. + */ +int +relay_decrypt_cell(circuit_t *circ, cell_t *cell, + cell_direction_t cell_direction, + crypt_path_t **layer_hint, char *recognized) +{ + relay_header_t rh; + + tor_assert(circ); + tor_assert(cell); + tor_assert(recognized); + tor_assert(cell_direction == CELL_DIRECTION_IN || + cell_direction == CELL_DIRECTION_OUT); + + if (cell_direction == CELL_DIRECTION_IN) { + if (CIRCUIT_IS_ORIGIN(circ)) { /* We're at the beginning of the circuit. + * We'll want to do layered decrypts. */ + crypt_path_t *thishop, *cpath = TO_ORIGIN_CIRCUIT(circ)->cpath; + thishop = cpath; + if (thishop->state != CPATH_STATE_OPEN) { + log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, + "Relay cell before first created cell? Closing."); + return -1; + } + do { /* Remember: cpath is in forward order, that is, first hop first. */ + tor_assert(thishop); + + /* decrypt one layer */ + relay_crypt_one_payload(thishop->crypto.b_crypto, cell->payload); + + relay_header_unpack(&rh, cell->payload); + if (rh.recognized == 0) { + /* it's possibly recognized. have to check digest to be sure. */ + if (relay_digest_matches(thishop->crypto.b_digest, cell)) { + *recognized = 1; + *layer_hint = thishop; + return 0; + } + } + + thishop = thishop->next; + } while (thishop != cpath && thishop->state == CPATH_STATE_OPEN); + log_fn(LOG_PROTOCOL_WARN, LD_OR, + "Incoming cell at client not recognized. Closing."); + return -1; + } else { + relay_crypto_t *crypto = &TO_OR_CIRCUIT(circ)->crypto; + /* We're in the middle. Encrypt one layer. */ + relay_crypt_one_payload(crypto->b_crypto, cell->payload); + } + } else /* cell_direction == CELL_DIRECTION_OUT */ { + /* We're in the middle. Decrypt one layer. */ + relay_crypto_t *crypto = &TO_OR_CIRCUIT(circ)->crypto; + + relay_crypt_one_payload(crypto->f_crypto, cell->payload); + + relay_header_unpack(&rh, cell->payload); + if (rh.recognized == 0) { + /* it's possibly recognized. have to check digest to be sure. */ + if (relay_digest_matches(crypto->f_digest, cell)) { + *recognized = 1; + return 0; + } + } + } + return 0; +} + +/** + * Encrypt a cell <b>cell</b> that we are creating, and sending outbound on + * <b>circ</b> until the hop corresponding to <b>layer_hint</b>. + * + * The integrity field and recognized field of <b>cell</b>'s relay headers + * must be set to zero. + */ +void +relay_encrypt_cell_outbound(cell_t *cell, + origin_circuit_t *circ, + crypt_path_t *layer_hint) +{ + crypt_path_t *thishop; /* counter for repeated crypts */ + relay_set_digest(layer_hint->crypto.f_digest, cell); + + thishop = layer_hint; + /* moving from farthest to nearest hop */ + do { + tor_assert(thishop); + log_debug(LD_OR,"encrypting a layer of the relay cell."); + relay_crypt_one_payload(thishop->crypto.f_crypto, cell->payload); + + thishop = thishop->prev; + } while (thishop != circ->cpath->prev); +} + +/** + * Encrypt a cell <b>cell</b> that we are creating, and sending on + * <b>circuit</b> to the origin. + * + * The integrity field and recognized field of <b>cell</b>'s relay headers + * must be set to zero. + */ +void +relay_encrypt_cell_inbound(cell_t *cell, + or_circuit_t *or_circ) +{ + relay_set_digest(or_circ->crypto.b_digest, cell); + /* encrypt one layer */ + relay_crypt_one_payload(or_circ->crypto.b_crypto, cell->payload); +} + +/** + * Release all storage held inside <b>crypto</b>, but do not free + * <b>crypto</b> itself: it lives inside another object. + */ +void +relay_crypto_clear(relay_crypto_t *crypto) +{ + if (BUG(!crypto)) + return; + crypto_cipher_free(crypto->f_crypto); + crypto_cipher_free(crypto->b_crypto); + crypto_digest_free(crypto->f_digest); + crypto_digest_free(crypto->b_digest); +} + +/** Initialize <b>crypto</b> from the key material in key_data. + * + * If <b>is_hs_v3</b> is set, this cpath will be used for next gen hidden + * service circuits and <b>key_data</b> must be at least + * HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN bytes in length. + * + * If <b>is_hs_v3</b> is not set, key_data must contain CPATH_KEY_MATERIAL_LEN + * bytes, which are used as follows: + * - 20 to initialize f_digest + * - 20 to initialize b_digest + * - 16 to key f_crypto + * - 16 to key b_crypto + * + * (If 'reverse' is true, then f_XX and b_XX are swapped.) + * + * Return 0 if init was successful, else -1 if it failed. + */ +int +relay_crypto_init(relay_crypto_t *crypto, + const char *key_data, size_t key_data_len, + int reverse, int is_hs_v3) +{ + crypto_digest_t *tmp_digest; + crypto_cipher_t *tmp_crypto; + size_t digest_len = 0; + size_t cipher_key_len = 0; + + tor_assert(crypto); + tor_assert(key_data); + tor_assert(!(crypto->f_crypto || crypto->b_crypto || + crypto->f_digest || crypto->b_digest)); + + /* Basic key size validation */ + if (is_hs_v3 && BUG(key_data_len != HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN)) { + goto err; + } else if (!is_hs_v3 && BUG(key_data_len != CPATH_KEY_MATERIAL_LEN)) { + goto err; + } + + /* If we are using this crypto for next gen onion services use SHA3-256, + otherwise use good ol' SHA1 */ + if (is_hs_v3) { + digest_len = DIGEST256_LEN; + cipher_key_len = CIPHER256_KEY_LEN; + crypto->f_digest = crypto_digest256_new(DIGEST_SHA3_256); + crypto->b_digest = crypto_digest256_new(DIGEST_SHA3_256); + } else { + digest_len = DIGEST_LEN; + cipher_key_len = CIPHER_KEY_LEN; + crypto->f_digest = crypto_digest_new(); + crypto->b_digest = crypto_digest_new(); + } + + tor_assert(digest_len != 0); + tor_assert(cipher_key_len != 0); + const int cipher_key_bits = (int) cipher_key_len * 8; + + crypto_digest_add_bytes(crypto->f_digest, key_data, digest_len); + crypto_digest_add_bytes(crypto->b_digest, key_data+digest_len, digest_len); + + crypto->f_crypto = crypto_cipher_new_with_bits(key_data+(2*digest_len), + cipher_key_bits); + if (!crypto->f_crypto) { + log_warn(LD_BUG,"Forward cipher initialization failed."); + goto err; + } + + crypto->b_crypto = crypto_cipher_new_with_bits( + key_data+(2*digest_len)+cipher_key_len, + cipher_key_bits); + if (!crypto->b_crypto) { + log_warn(LD_BUG,"Backward cipher initialization failed."); + goto err; + } + + if (reverse) { + tmp_digest = crypto->f_digest; + crypto->f_digest = crypto->b_digest; + crypto->b_digest = tmp_digest; + tmp_crypto = crypto->f_crypto; + crypto->f_crypto = crypto->b_crypto; + crypto->b_crypto = tmp_crypto; + } + + return 0; + err: + relay_crypto_clear(crypto); + return -1; +} + +/** Assert that <b>crypto</b> is valid and set. */ +void +relay_crypto_assert_ok(const relay_crypto_t *crypto) +{ + tor_assert(crypto->f_crypto); + tor_assert(crypto->b_crypto); + tor_assert(crypto->f_digest); + tor_assert(crypto->b_digest); +} + diff --git a/src/or/relay_crypto.h b/src/or/relay_crypto.h new file mode 100644 index 0000000000..66ae02cee9 --- /dev/null +++ b/src/or/relay_crypto.h @@ -0,0 +1,31 @@ +/* Copyright (c) 2001 Matej Pfajfar. + * Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2017, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file relay.h + * \brief Header file for relay.c. + **/ + +#ifndef TOR_RELAY_CRYPTO_H +#define TOR_RELAY_CRYPTO_H + +int relay_crypto_init(relay_crypto_t *crypto, + const char *key_data, size_t key_data_len, + int reverse, int is_hs_v3); + +int relay_decrypt_cell(circuit_t *circ, cell_t *cell, + cell_direction_t cell_direction, + crypt_path_t **layer_hint, char *recognized); +void relay_encrypt_cell_outbound(cell_t *cell, origin_circuit_t *or_circ, + crypt_path_t *layer_hint); +void relay_encrypt_cell_inbound(cell_t *cell, or_circuit_t *or_circ); + +void relay_crypto_clear(relay_crypto_t *crypto); + +void relay_crypto_assert_ok(const relay_crypto_t *crypto); + +#endif /* !defined(TOR_RELAY_CRYPTO_H) */ + diff --git a/src/or/rendclient.c b/src/or/rendclient.c index e61207f2ee..74118f8f37 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -15,10 +15,13 @@ #include "config.h" #include "connection.h" #include "connection_edge.h" +#include "control.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "directory.h" -#include "hs_common.h" #include "hs_circuit.h" #include "hs_client.h" +#include "hs_common.h" #include "main.h" #include "networkstatus.h" #include "nodelist.h" @@ -29,7 +32,6 @@ #include "router.h" #include "routerlist.h" #include "routerset.h" -#include "control.h" static extend_info_t *rend_client_get_random_intro_impl( const rend_cache_entry_t *rend_query, @@ -924,8 +926,8 @@ rend_client_desc_trynow(const char *query) /* restart their timeout values, so they get a fair shake at * connecting to the hidden service. */ base_conn->timestamp_created = now; - base_conn->timestamp_lastread = now; - base_conn->timestamp_lastwritten = now; + base_conn->timestamp_last_read_allowed = now; + base_conn->timestamp_last_write_allowed = now; connection_ap_mark_as_pending_circuit(conn); } else { /* 404, or fetch didn't get that far */ diff --git a/src/or/rendcommon.c b/src/or/rendcommon.c index 230da4be5c..f3fa2f64d1 100644 --- a/src/or/rendcommon.c +++ b/src/or/rendcommon.c @@ -12,20 +12,23 @@ #include "or.h" #include "circuitbuild.h" +#include "circuituse.h" #include "config.h" #include "control.h" +#include "crypto_rand.h" +#include "crypto_util.h" +#include "hs_client.h" #include "hs_common.h" +#include "hs_intropoint.h" +#include "networkstatus.h" #include "rendclient.h" #include "rendcommon.h" #include "rendmid.h" -#include "hs_intropoint.h" -#include "hs_client.h" #include "rendservice.h" #include "rephist.h" #include "router.h" #include "routerlist.h" #include "routerparse.h" -#include "networkstatus.h" /** Return 0 if one and two are the same service ids, else -1 or 1 */ int @@ -807,6 +810,11 @@ rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint, tor_fragile_assert(); } + if (r == 0 && origin_circ) { + /* This was a valid cell. Count it as delivered + overhead. */ + circuit_read_valid_data(origin_circ, length); + } + if (r == -2) log_info(LD_PROTOCOL, "Dropping cell (type %d) for wrong circuit type.", command); diff --git a/src/or/rendservice.c b/src/or/rendservice.c index ac86c143d1..92c323b10d 100644 --- a/src/or/rendservice.c +++ b/src/or/rendservice.c @@ -16,6 +16,8 @@ #include "circuituse.h" #include "config.h" #include "control.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "directory.h" #include "hs_common.h" #include "hs_config.h" @@ -348,6 +350,13 @@ rend_add_service(smartlist_t *service_list, rend_service_t *service) /* The service passed all the checks */ tor_assert(s_list); smartlist_add(s_list, service); + + /* Notify that our global service list has changed only if this new service + * went into our global list. If not, when we move service from the staging + * list to the new list, a notify is triggered. */ + if (s_list == rend_service_list) { + hs_service_map_has_changed(); + } return 0; } @@ -609,6 +618,8 @@ rend_service_prune_list_impl_(void) circuit_mark_for_close(TO_CIRCUIT(ocirc), END_CIRC_REASON_FINISHED); } smartlist_free(surviving_services); + /* Notify that our global service list has changed. */ + hs_service_map_has_changed(); } /* Try to prune our main service list using the temporary one that we just @@ -959,6 +970,8 @@ rend_service_del_ephemeral(const char *service_id) } } SMARTLIST_FOREACH_END(circ); smartlist_remove(rend_service_list, s); + /* Notify that we just removed a service from our global list. */ + hs_service_map_has_changed(); rend_service_free(s); log_debug(LD_CONFIG, "Removed ephemeral Onion Service: %s", service_id); diff --git a/src/or/rephist.c b/src/or/rephist.c index 43494692cb..c7117bad63 100644 --- a/src/or/rephist.c +++ b/src/or/rephist.c @@ -78,6 +78,7 @@ #include "circuitlist.h" #include "circuituse.h" #include "config.h" +#include "crypto_rand.h" #include "networkstatus.h" #include "nodelist.h" #include "rephist.h" @@ -85,9 +86,8 @@ #include "routerlist.h" #include "ht.h" #include "channelpadding.h" - -#include "channelpadding.h" #include "connection_or.h" +#include "statefile.h" static void bw_arrays_init(void); static void predicted_ports_alloc(void); diff --git a/src/or/router.c b/src/or/router.c index a3d7cd373c..5485ec913e 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -13,6 +13,8 @@ #include "config.h" #include "connection.h" #include "control.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "crypto_curve25519.h" #include "directory.h" #include "dirserv.h" @@ -35,6 +37,8 @@ #include "transports.h" #include "routerset.h" +#include "dirauth/mode.h" + /** * \file router.c * \brief Miscellaneous relay functionality, including RSA key maintenance, @@ -103,6 +107,64 @@ static authority_cert_t *legacy_key_certificate = NULL; * used by tor-gencert to sign new signing keys and make new key * certificates. */ +const char *format_node_description(char *buf, + const char *id_digest, + int is_named, + const char *nickname, + const tor_addr_t *addr, + uint32_t addr32h); + +/** Return a readonly string with human readable description + * of <b>err</b>. + */ +const char * +routerinfo_err_to_string(int err) +{ + switch (err) { + case TOR_ROUTERINFO_ERROR_NO_EXT_ADDR: + return "No known exit address yet"; + case TOR_ROUTERINFO_ERROR_CANNOT_PARSE: + return "Cannot parse descriptor"; + case TOR_ROUTERINFO_ERROR_NOT_A_SERVER: + return "Not running in server mode"; + case TOR_ROUTERINFO_ERROR_DIGEST_FAILED: + return "Key digest failed"; + case TOR_ROUTERINFO_ERROR_CANNOT_GENERATE: + return "Cannot generate descriptor"; + case TOR_ROUTERINFO_ERROR_DESC_REBUILDING: + return "Descriptor still rebuilding - not ready yet"; + } + + log_warn(LD_BUG, "unknown routerinfo error %d - shouldn't happen", err); + tor_assert_unreached(); + + return "Unknown error"; +} + +/** Return true if we expect given error to be transient. + * Return false otherwise. + */ +int +routerinfo_err_is_transient(int err) +{ + switch (err) { + case TOR_ROUTERINFO_ERROR_NO_EXT_ADDR: + return 1; + case TOR_ROUTERINFO_ERROR_CANNOT_PARSE: + return 1; + case TOR_ROUTERINFO_ERROR_NOT_A_SERVER: + return 0; + case TOR_ROUTERINFO_ERROR_DIGEST_FAILED: + return 0; // XXX: bug? + case TOR_ROUTERINFO_ERROR_CANNOT_GENERATE: + return 1; + case TOR_ROUTERINFO_ERROR_DESC_REBUILDING: + return 1; + } + + return 0; +} + /** Replace the current onion key with <b>k</b>. Does not affect * lastonionkey; to update lastonionkey correctly, call rotate_onion_key(). */ @@ -1234,7 +1296,8 @@ check_whether_dirport_reachable(const or_options_t *options) /* XXX Should this be increased? */ #define MIN_BW_TO_ADVERTISE_DIRSERVER 51200 -/** Return true iff we have enough configured bandwidth to cache directory +/** Return true iff we have enough configured bandwidth to advertise or + * automatically provide directory services from cache directory * information. */ static int router_has_bandwidth_to_be_dirserver(const or_options_t *options) @@ -1257,7 +1320,7 @@ router_has_bandwidth_to_be_dirserver(const or_options_t *options) * MIN_BW_TO_ADVERTISE_DIRSERVER, don't bother trying to serve requests. */ static int -router_should_be_directory_server(const or_options_t *options, int dir_port) +router_should_be_dirserver(const or_options_t *options, int dir_port) { static int advertising=1; /* start out assuming we will advertise */ int new_choice=1; @@ -1362,7 +1425,7 @@ decide_to_advertise_dir_impl(const or_options_t *options, /* Part two: consider config options that could make us choose to * publish or not publish that the user might find surprising. */ - return router_should_be_directory_server(options, dir_port); + return router_should_be_dirserver(options, dir_port); } /** Front-end to decide_to_advertise_dir_impl(): return 0 if we don't want to @@ -1370,7 +1433,7 @@ decide_to_advertise_dir_impl(const or_options_t *options, * DirPort we want to advertise. */ static int -decide_to_advertise_dirport(const or_options_t *options, uint16_t dir_port) +router_should_advertise_dirport(const or_options_t *options, uint16_t dir_port) { /* supports_tunnelled_dir_requests is not relevant, pass 0 */ return decide_to_advertise_dir_impl(options, dir_port, 0) ? dir_port : 0; @@ -1380,7 +1443,7 @@ decide_to_advertise_dirport(const or_options_t *options, uint16_t dir_port) * advertise the fact that we support begindir requests, else return 1. */ static int -decide_to_advertise_begindir(const or_options_t *options, +router_should_advertise_begindir(const or_options_t *options, int supports_tunnelled_dir_requests) { /* dir_port is not relevant, pass 0 */ @@ -1413,26 +1476,17 @@ extend_info_from_router(const routerinfo_t *r) &ap.addr, ap.port); } -/** Some time has passed, or we just got new directory information. - * See if we currently believe our ORPort or DirPort to be - * unreachable. If so, launch a new test for it. - * - * For ORPort, we simply try making a circuit that ends at ourselves. - * Success is noticed in onionskin_answer(). - * - * For DirPort, we make a connection via Tor to our DirPort and ask - * for our own server descriptor. - * Success is noticed in connection_dir_client_reached_eof(). +/**See if we currently believe our ORPort or DirPort to be + * unreachable. If so, return 1 else return 0. */ -void -consider_testing_reachability(int test_or, int test_dir) +static int +router_should_check_reachability(int test_or, int test_dir) { const routerinfo_t *me = router_get_my_routerinfo(); const or_options_t *options = get_options(); - int orport_reachable = check_whether_orport_reachable(options); - tor_addr_t addr; + if (!me) - return; + return 0; if (routerset_contains_router(options->ExcludeNodes, me, -1) && options->StrictNodes) { @@ -1447,43 +1501,66 @@ consider_testing_reachability(int test_or, int test_dir) "We cannot learn whether we are usable, and will not " "be able to advertise ourself."); } - return; + return 0; } + return 1; +} + +/** Some time has passed, or we just got new directory information. + * See if we currently believe our ORPort or DirPort to be + * unreachable. If so, launch a new test for it. + * + * For ORPort, we simply try making a circuit that ends at ourselves. + * Success is noticed in onionskin_answer(). + * + * For DirPort, we make a connection via Tor to our DirPort and ask + * for our own server descriptor. + * Success is noticed in connection_dir_client_reached_eof(). + */ +void +router_do_reachability_checks(int test_or, int test_dir) +{ + const routerinfo_t *me = router_get_my_routerinfo(); + const or_options_t *options = get_options(); + int orport_reachable = check_whether_orport_reachable(options); + tor_addr_t addr; + + if (router_should_check_reachability(test_or, test_dir)) { + if (test_or && (!orport_reachable || !circuit_enough_testing_circs())) { + extend_info_t *ei = extend_info_from_router(me); + /* XXX IPv6 self testing */ + log_info(LD_CIRC, "Testing %s of my ORPort: %s:%d.", + !orport_reachable ? "reachability" : "bandwidth", + fmt_addr32(me->addr), me->or_port); + circuit_launch_by_extend_info(CIRCUIT_PURPOSE_TESTING, ei, + CIRCLAUNCH_NEED_CAPACITY|CIRCLAUNCH_IS_INTERNAL); + extend_info_free(ei); + } - if (test_or && (!orport_reachable || !circuit_enough_testing_circs())) { - extend_info_t *ei = extend_info_from_router(me); /* XXX IPv6 self testing */ - log_info(LD_CIRC, "Testing %s of my ORPort: %s:%d.", - !orport_reachable ? "reachability" : "bandwidth", - fmt_addr32(me->addr), me->or_port); - circuit_launch_by_extend_info(CIRCUIT_PURPOSE_TESTING, ei, - CIRCLAUNCH_NEED_CAPACITY|CIRCLAUNCH_IS_INTERNAL); - extend_info_free(ei); - } - - /* XXX IPv6 self testing */ - tor_addr_from_ipv4h(&addr, me->addr); - if (test_dir && !check_whether_dirport_reachable(options) && - !connection_get_by_type_addr_port_purpose( - CONN_TYPE_DIR, &addr, me->dir_port, - DIR_PURPOSE_FETCH_SERVERDESC)) { - tor_addr_port_t my_orport, my_dirport; - memcpy(&my_orport.addr, &addr, sizeof(addr)); - memcpy(&my_dirport.addr, &addr, sizeof(addr)); - my_orport.port = me->or_port; - my_dirport.port = me->dir_port; - /* ask myself, via tor, for my server descriptor. */ - directory_request_t *req = - directory_request_new(DIR_PURPOSE_FETCH_SERVERDESC); - directory_request_set_or_addr_port(req, &my_orport); - directory_request_set_dir_addr_port(req, &my_dirport); - directory_request_set_directory_id_digest(req, + tor_addr_from_ipv4h(&addr, me->addr); + if (test_dir && !check_whether_dirport_reachable(options) && + !connection_get_by_type_addr_port_purpose( + CONN_TYPE_DIR, &addr, me->dir_port, + DIR_PURPOSE_FETCH_SERVERDESC)) { + tor_addr_port_t my_orport, my_dirport; + memcpy(&my_orport.addr, &addr, sizeof(addr)); + memcpy(&my_dirport.addr, &addr, sizeof(addr)); + my_orport.port = me->or_port; + my_dirport.port = me->dir_port; + /* ask myself, via tor, for my server descriptor. */ + directory_request_t *req = + directory_request_new(DIR_PURPOSE_FETCH_SERVERDESC); + directory_request_set_or_addr_port(req, &my_orport); + directory_request_set_dir_addr_port(req, &my_dirport); + directory_request_set_directory_id_digest(req, me->cache_info.identity_digest); - // ask via an anon circuit, connecting to our dirport. - directory_request_set_indirection(req, DIRIND_ANON_DIRPORT); - directory_request_set_resource(req, "authority.z"); - directory_initiate_request(req); - directory_request_free(req); + // ask via an anon circuit, connecting to our dirport. + directory_request_set_indirection(req, DIRIND_ANON_DIRPORT); + directory_request_set_resource(req, "authority.z"); + directory_initiate_request(req); + directory_request_free(req); + } } } @@ -1528,7 +1605,7 @@ router_dirport_found_reachable(void) && check_whether_orport_reachable(options) ? " Publishing server descriptor." : ""); can_reach_dir_port = 1; - if (decide_to_advertise_dirport(options, me->dir_port)) { + if (router_should_advertise_dirport(options, me->dir_port)) { mark_my_descriptor_dirty("DirPort found reachable"); /* This is a significant enough change to upload immediately, * at least in a test network */ @@ -1573,30 +1650,31 @@ router_perform_bandwidth_test(int num_circs, time_t now) } } -/** Return true iff our network is in some sense disabled: either we're - * hibernating, entering hibernation, or the network is turned off with - * DisableNetwork. */ +/** Return true iff our network is in some sense disabled or shutting down: + * either we're hibernating, entering hibernation, or the network is turned + * off with DisableNetwork. */ int net_is_disabled(void) { return get_options()->DisableNetwork || we_are_hibernating(); } -/** Return true iff we believe ourselves to be an authoritative - * directory server. - */ +/** Return true iff our network is in some sense "completely disabled" either + * we're fully hibernating or the network is turned off with + * DisableNetwork. */ int -authdir_mode(const or_options_t *options) +net_is_completely_disabled(void) { - return options->AuthoritativeDir != 0; + return get_options()->DisableNetwork || we_are_fully_hibernating(); } -/** Return true iff we believe ourselves to be a v3 authoritative + +/** Return true iff we believe ourselves to be an authoritative * directory server. */ int -authdir_mode_v3(const or_options_t *options) +authdir_mode(const or_options_t *options) { - return authdir_mode(options) && options->V3AuthoritativeDir != 0; + return options->AuthoritativeDir != 0; } /** Return true iff we are an authoritative directory server that is * authoritative about receiving and serving descriptors of type @@ -1999,10 +2077,43 @@ router_is_me(const routerinfo_t *router) MOCK_IMPL(const routerinfo_t *, router_get_my_routerinfo,(void)) { - if (!server_mode(get_options())) + return router_get_my_routerinfo_with_err(NULL); +} + +/** Return routerinfo of this OR. Rebuild it from + * scratch if needed. Set <b>*err</b> to 0 on success or to + * appropriate TOR_ROUTERINFO_ERROR_* value on failure. + */ +MOCK_IMPL(const routerinfo_t *, +router_get_my_routerinfo_with_err,(int *err)) +{ + if (!server_mode(get_options())) { + if (err) + *err = TOR_ROUTERINFO_ERROR_NOT_A_SERVER; + return NULL; - if (router_rebuild_descriptor(0)) + } + + if (!desc_clean_since) { + int rebuild_err = router_rebuild_descriptor(0); + if (rebuild_err < 0) { + if (err) + *err = rebuild_err; + + return NULL; + } + } + + if (!desc_routerinfo) { + if (err) + *err = TOR_ROUTERINFO_ERROR_DESC_REBUILDING; + return NULL; + } + + if (err) + *err = 0; + return desc_routerinfo; } @@ -2179,7 +2290,7 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) if (router_pick_published_address(options, &addr, 0) < 0) { log_warn(LD_CONFIG, "Don't know my address while generating descriptor"); - return -1; + return TOR_ROUTERINFO_ERROR_NO_EXT_ADDR; } /* Log a message if the address in the descriptor doesn't match the ORPort @@ -2235,7 +2346,7 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) if (crypto_pk_get_digest(ri->identity_pkey, ri->cache_info.identity_digest)<0) { routerinfo_free(ri); - return -1; + return TOR_ROUTERINFO_ERROR_DIGEST_FAILED; } ri->cache_info.signing_key_cert = tor_cert_dup(get_master_signing_key_cert()); @@ -2251,6 +2362,7 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) /* and compute ri->bandwidthburst similarly */ ri->bandwidthburst = get_effective_bwburst(options); + /* Report bandwidth, unless we're hibernating or shutting down */ ri->bandwidthcapacity = hibernating ? 0 : rep_hist_bandwidth_assess(); if (dns_seems_to_be_broken() || has_dns_init_failed()) { @@ -2368,7 +2480,7 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) log_warn(LD_BUG, "Couldn't generate router descriptor."); routerinfo_free(ri); extrainfo_free(ei); - return -1; + return TOR_ROUTERINFO_ERROR_CANNOT_GENERATE; } ri->cache_info.signed_descriptor_len = strlen(ri->cache_info.signed_descriptor_body); @@ -2411,6 +2523,7 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e) int router_rebuild_descriptor(int force) { + int err = 0; routerinfo_t *ri; extrainfo_t *ei; uint32_t addr; @@ -2425,13 +2538,14 @@ router_rebuild_descriptor(int force) * learn that it's time to try again when ip_address_changed() * marks it dirty. */ desc_clean_since = time(NULL); - return -1; + return TOR_ROUTERINFO_ERROR_DESC_REBUILDING; } log_info(LD_OR, "Rebuilding relay descriptor%s", force ? " (forced)" : ""); - if (router_build_fresh_descriptor(&ri, &ei) < 0) { - return -1; + err = router_build_fresh_descriptor(&ri, &ei); + if (err < 0) { + return err; } routerinfo_free(desc_routerinfo); @@ -2528,6 +2642,9 @@ check_descriptor_bandwidth_changed(time_t now) return; prev = my_ri->bandwidthcapacity; + + /* Consider ourselves to have zero bandwidth if we're hibernating or + * shutting down. */ cur = we_are_hibernating() ? 0 : rep_hist_bandwidth_assess(); if ((prev != cur && (!prev || !cur)) || cur > prev*2 || @@ -2931,14 +3048,14 @@ router_dump_router_to_string(routerinfo_t *router, router->nickname, address, router->or_port, - decide_to_advertise_dirport(options, router->dir_port), + router_should_advertise_dirport(options, router->dir_port), ed_cert_line ? ed_cert_line : "", extra_or_address ? extra_or_address : "", router->platform, proto_line, published, fingerprint, - stats_n_seconds_working, + get_uptime(), (int) router->bandwidthrate, (int) router->bandwidthburst, (int) router->bandwidthcapacity, @@ -3005,7 +3122,7 @@ router_dump_router_to_string(routerinfo_t *router, tor_free(p6); } - if (decide_to_advertise_begindir(options, + if (router_should_advertise_begindir(options, router->supports_tunnelled_dir_requests)) { smartlist_add_strdup(chunks, "tunnelled-dir-server\n"); } @@ -3454,6 +3571,15 @@ is_legal_hexdigest(const char *s) strspn(s,HEX_CHARACTERS)==HEX_DIGEST_LEN); } +/** + * Longest allowed output of format_node_description, plus 1 character for + * NUL. This allows space for: + * "$FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF~xxxxxxxxxxxxxxxxxxx at" + * " [ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255]" + * plus a terminating NUL. + */ +#define NODE_DESC_BUF_LEN (MAX_VERBOSE_NICKNAME_LEN+4+TOR_ADDR_BUF_LEN) + /** Use <b>buf</b> (which must be at least NODE_DESC_BUF_LEN bytes long) to * hold a human-readable description of a node with identity digest * <b>id_digest</b>, named-status <b>is_named</b>, nickname <b>nickname</b>, @@ -3499,15 +3625,16 @@ format_node_description(char *buf, return buf; } -/** Use <b>buf</b> (which must be at least NODE_DESC_BUF_LEN bytes long) to - * hold a human-readable description of <b>ri</b>. - * +/** Return a human-readable description of the routerinfo_t <b>ri</b>. * - * Return a pointer to the front of <b>buf</b>. + * This function is not thread-safe. Each call to this function invalidates + * previous values returned by this function. */ const char * -router_get_description(char *buf, const routerinfo_t *ri) +router_describe(const routerinfo_t *ri) { + static char buf[NODE_DESC_BUF_LEN]; + if (!ri) return "<null>"; return format_node_description(buf, @@ -3518,14 +3645,15 @@ router_get_description(char *buf, const routerinfo_t *ri) ri->addr); } -/** Use <b>buf</b> (which must be at least NODE_DESC_BUF_LEN bytes long) to - * hold a human-readable description of <b>node</b>. +/** Return a human-readable description of the node_t <b>node</b>. * - * Return a pointer to the front of <b>buf</b>. + * This function is not thread-safe. Each call to this function invalidates + * previous values returned by this function. */ const char * -node_get_description(char *buf, const node_t *node) +node_describe(const node_t *node) { + static char buf[NODE_DESC_BUF_LEN]; const char *nickname = NULL; uint32_t addr32h = 0; int is_named = 0; @@ -3550,14 +3678,16 @@ node_get_description(char *buf, const node_t *node) addr32h); } -/** Use <b>buf</b> (which must be at least NODE_DESC_BUF_LEN bytes long) to - * hold a human-readable description of <b>rs</b>. +/** Return a human-readable description of the routerstatus_t <b>rs</b>. * - * Return a pointer to the front of <b>buf</b>. + * This function is not thread-safe. Each call to this function invalidates + * previous values returned by this function. */ const char * -routerstatus_get_description(char *buf, const routerstatus_t *rs) +routerstatus_describe(const routerstatus_t *rs) { + static char buf[NODE_DESC_BUF_LEN]; + if (!rs) return "<null>"; return format_node_description(buf, @@ -3568,14 +3698,16 @@ routerstatus_get_description(char *buf, const routerstatus_t *rs) rs->addr); } -/** Use <b>buf</b> (which must be at least NODE_DESC_BUF_LEN bytes long) to - * hold a human-readable description of <b>ei</b>. +/** Return a human-readable description of the extend_info_t <b>ei</b>. * - * Return a pointer to the front of <b>buf</b>. + * This function is not thread-safe. Each call to this function invalidates + * previous values returned by this function. */ const char * -extend_info_get_description(char *buf, const extend_info_t *ei) +extend_info_describe(const extend_info_t *ei) { + static char buf[NODE_DESC_BUF_LEN]; + if (!ei) return "<null>"; return format_node_description(buf, @@ -3586,54 +3718,6 @@ extend_info_get_description(char *buf, const extend_info_t *ei) 0); } -/** Return a human-readable description of the routerinfo_t <b>ri</b>. - * - * This function is not thread-safe. Each call to this function invalidates - * previous values returned by this function. - */ -const char * -router_describe(const routerinfo_t *ri) -{ - static char buf[NODE_DESC_BUF_LEN]; - return router_get_description(buf, ri); -} - -/** Return a human-readable description of the node_t <b>node</b>. - * - * This function is not thread-safe. Each call to this function invalidates - * previous values returned by this function. - */ -const char * -node_describe(const node_t *node) -{ - static char buf[NODE_DESC_BUF_LEN]; - return node_get_description(buf, node); -} - -/** Return a human-readable description of the routerstatus_t <b>rs</b>. - * - * This function is not thread-safe. Each call to this function invalidates - * previous values returned by this function. - */ -const char * -routerstatus_describe(const routerstatus_t *rs) -{ - static char buf[NODE_DESC_BUF_LEN]; - return routerstatus_get_description(buf, rs); -} - -/** Return a human-readable description of the extend_info_t <b>ei</b>. - * - * This function is not thread-safe. Each call to this function invalidates - * previous values returned by this function. - */ -const char * -extend_info_describe(const extend_info_t *ei) -{ - static char buf[NODE_DESC_BUF_LEN]; - return extend_info_get_description(buf, ei); -} - /** Set <b>buf</b> (which must have MAX_VERBOSE_NICKNAME_LEN+1 bytes) to the * verbose representation of the identity of <b>router</b>. The format is: * A dollar sign. @@ -3733,4 +3817,3 @@ router_get_all_orports(const routerinfo_t *ri) fake_node.ri = (routerinfo_t *)ri; return node_get_all_orports(&fake_node); } - diff --git a/src/or/router.h b/src/or/router.h index 696e983662..752f2f2dbe 100644 --- a/src/or/router.h +++ b/src/or/router.h @@ -14,6 +14,13 @@ #include "testsupport.h" +#define TOR_ROUTERINFO_ERROR_NO_EXT_ADDR (-1) +#define TOR_ROUTERINFO_ERROR_CANNOT_PARSE (-2) +#define TOR_ROUTERINFO_ERROR_NOT_A_SERVER (-3) +#define TOR_ROUTERINFO_ERROR_DIGEST_FAILED (-4) +#define TOR_ROUTERINFO_ERROR_CANNOT_GENERATE (-5) +#define TOR_ROUTERINFO_ERROR_DESC_REBUILDING (-6) + crypto_pk_t *get_onion_key(void); time_t get_onion_key_set_at(void); void set_server_identity_key(crypto_pk_t *k); @@ -47,15 +54,15 @@ int init_keys_client(void); int check_whether_orport_reachable(const or_options_t *options); int check_whether_dirport_reachable(const or_options_t *options); int dir_server_mode(const or_options_t *options); -void consider_testing_reachability(int test_or, int test_dir); +void router_do_reachability_checks(int test_or, int test_dir); void router_orport_found_reachable(void); void router_dirport_found_reachable(void); void router_perform_bandwidth_test(int num_circs, time_t now); int net_is_disabled(void); +int net_is_completely_disabled(void); int authdir_mode(const or_options_t *options); -int authdir_mode_v3(const or_options_t *options); int authdir_mode_handles_descs(const or_options_t *options, int purpose); int authdir_mode_publishes_statuses(const or_options_t *options); int authdir_mode_tests_reachability(const or_options_t *options); @@ -86,6 +93,7 @@ void router_new_address_suggestion(const char *suggestion, int router_compare_to_my_exit_policy(const tor_addr_t *addr, uint16_t port); MOCK_DECL(int, router_my_exit_policy_is_reject_star,(void)); MOCK_DECL(const routerinfo_t *, router_get_my_routerinfo, (void)); +MOCK_DECL(const routerinfo_t *, router_get_my_routerinfo_with_err,(int *err)); extrainfo_t *router_get_my_extrainfo(void); const char *router_get_my_descriptor(void); const char *router_get_descriptor_gen_reason(void); @@ -123,29 +131,14 @@ int is_legal_nickname(const char *s); int is_legal_nickname_or_hexdigest(const char *s); int is_legal_hexdigest(const char *s); -/** - * Longest allowed output of format_node_description, plus 1 character for - * NUL. This allows space for: - * "$FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF~xxxxxxxxxxxxxxxxxxx at" - * " [ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255]" - * plus a terminating NUL. - */ -#define NODE_DESC_BUF_LEN (MAX_VERBOSE_NICKNAME_LEN+4+TOR_ADDR_BUF_LEN) -const char *format_node_description(char *buf, - const char *id_digest, - int is_named, - const char *nickname, - const tor_addr_t *addr, - uint32_t addr32h); -const char *router_get_description(char *buf, const routerinfo_t *ri); -const char *node_get_description(char *buf, const node_t *node); -const char *routerstatus_get_description(char *buf, const routerstatus_t *rs); -const char *extend_info_get_description(char *buf, const extend_info_t *ei); const char *router_describe(const routerinfo_t *ri); const char *node_describe(const node_t *node); const char *routerstatus_describe(const routerstatus_t *ri); const char *extend_info_describe(const extend_info_t *ei); +const char *routerinfo_err_to_string(int err); +int routerinfo_err_is_transient(int err); + void router_get_verbose_nickname(char *buf, const routerinfo_t *router); void router_reset_warnings(void); void router_reset_reachability(void); diff --git a/src/or/routerkeys.c b/src/or/routerkeys.c index 1933aaf4b6..43460da8cc 100644 --- a/src/or/routerkeys.c +++ b/src/or/routerkeys.c @@ -16,6 +16,7 @@ #include "or.h" #include "config.h" +#include "crypto_util.h" #include "router.h" #include "crypto_pwbox.h" #include "routerkeys.h" diff --git a/src/or/routerlist.c b/src/or/routerlist.c index 203895c867..2f27af7f06 100644 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@ -99,9 +99,9 @@ #include "config.h" #include "connection.h" #include "control.h" +#include "crypto_rand.h" #include "directory.h" #include "dirserv.h" -#include "dirvote.h" #include "entrynodes.h" #include "fp_pair.h" #include "geoip.h" @@ -122,6 +122,9 @@ #include "sandbox.h" #include "torcert.h" +#include "dirauth/dirvote.h" +#include "dirauth/mode.h" + // #define DEBUG_ROUTERLIST /****************************************************************************/ @@ -2761,9 +2764,8 @@ frac_nodes_with_descriptors(const smartlist_t *sl, if (node_has_any_descriptor(node)) n_with_descs++; }); - tor_free(bandwidths); - return ((double)n_with_descs) / (double)smartlist_len(sl); + return ((double)n_with_descs) / smartlist_len(sl); } present = 0.0; diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 6f82859e61..e82ecec5b7 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -56,30 +56,34 @@ #define ROUTERPARSE_PRIVATE #include "or.h" -#include "config.h" #include "circuitstats.h" +#include "config.h" +#include "crypto_util.h" +#include "dirauth/shared_random.h" #include "dirserv.h" -#include "dirvote.h" +#include "entrynodes.h" +#include "memarea.h" +#include "microdesc.h" +#include "networkstatus.h" #include "parsecommon.h" #include "policies.h" #include "protover.h" #include "rendcommon.h" -#include "router.h" -#include "routerlist.h" -#include "memarea.h" -#include "microdesc.h" -#include "networkstatus.h" #include "rephist.h" +#include "router.h" #include "routerkeys.h" +#include "routerlist.h" #include "routerparse.h" -#include "entrynodes.h" -#include "torcert.h" #include "sandbox.h" -#include "shared_random.h" +#include "shared_random_client.h" +#include "torcert.h" +#include "voting_schedule.h" #undef log #include <math.h> +#include "dirauth/dirvote.h" + /****************************************************************************/ /** List of tokens recognized in router descriptors */ @@ -2743,8 +2747,7 @@ routerstatus_parse_entry_from_string(memarea_t *area, /* These are implied true by having been included in a consensus made * with a given method */ rs->is_flagged_running = 1; /* Starting with consensus method 4. */ - if (consensus_method >= MIN_METHOD_FOR_EXCLUDING_INVALID_NODES) - rs->is_valid = 1; + rs->is_valid = 1; /* Starting with consensus method 24. */ } { const char *protocols = NULL, *version = NULL; @@ -3283,60 +3286,6 @@ networkstatus_verify_bw_weights(networkstatus_t *ns, int consensus_method) return valid; } -/** Parse and extract all SR commits from <b>tokens</b> and place them in - * <b>ns</b>. */ -static void -extract_shared_random_commits(networkstatus_t *ns, smartlist_t *tokens) -{ - smartlist_t *chunks = NULL; - - tor_assert(ns); - tor_assert(tokens); - /* Commits are only present in a vote. */ - tor_assert(ns->type == NS_TYPE_VOTE); - - ns->sr_info.commits = smartlist_new(); - - smartlist_t *commits = find_all_by_keyword(tokens, K_COMMIT); - /* It's normal that a vote might contain no commits even if it participates - * in the SR protocol. Don't treat it as an error. */ - if (commits == NULL) { - goto end; - } - - /* Parse the commit. We do NO validation of number of arguments or ordering - * for forward compatibility, it's the parse commit job to inform us if it's - * supported or not. */ - chunks = smartlist_new(); - SMARTLIST_FOREACH_BEGIN(commits, directory_token_t *, tok) { - /* Extract all arguments and put them in the chunks list. */ - for (int i = 0; i < tok->n_args; i++) { - smartlist_add(chunks, tok->args[i]); - } - sr_commit_t *commit = sr_parse_commit(chunks); - smartlist_clear(chunks); - if (commit == NULL) { - /* Get voter identity so we can warn that this dirauth vote contains - * commit we can't parse. */ - networkstatus_voter_info_t *voter = smartlist_get(ns->voters, 0); - tor_assert(voter); - log_warn(LD_DIR, "SR: Unable to parse commit %s from vote of voter %s.", - escaped(tok->object_body), - hex_str(voter->identity_digest, - sizeof(voter->identity_digest))); - /* Commitment couldn't be parsed. Continue onto the next commit because - * this one could be unsupported for instance. */ - continue; - } - /* Add newly created commit object to the vote. */ - smartlist_add(ns->sr_info.commits, commit); - } SMARTLIST_FOREACH_END(tok); - - end: - smartlist_free(chunks); - smartlist_free(commits); -} - /** Check if a shared random value of type <b>srv_type</b> is in * <b>tokens</b>. If there is, parse it and set it to <b>srv_out</b>. Return * -1 on failure, 0 on success. The resulting srv is allocated on the heap and @@ -3774,13 +3723,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, /* If this is a vote document, check if information about the shared randomness protocol is included, and extract it. */ if (ns->type == NS_TYPE_VOTE) { - /* Does this authority participates in the SR protocol? */ - tok = find_opt_by_keyword(tokens, K_SR_FLAG); - if (tok) { - ns->sr_info.participate = 1; - /* Get the SR commitments and reveals from the vote. */ - extract_shared_random_commits(ns, tokens); - } + dirvote_parse_sr_commits(ns, tokens); } /* For both a vote and consensus, extract the shared random values. */ if (ns->type == NS_TYPE_VOTE || ns->type == NS_TYPE_CONSENSUS) { @@ -3970,7 +3913,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out, } } - if (voter_get_sig_by_algorithm(v, sig->alg)) { + if (networkstatus_get_voter_sig_by_alg(v, sig->alg)) { /* We already parsed a vote with this algorithm from this voter. Use the first one. */ log_fn(LOG_PROTOCOL_WARN, LD_DIR, "We received a networkstatus " diff --git a/src/or/scheduler.c b/src/or/scheduler.c index 382b3e3ca9..da894294bf 100644 --- a/src/or/scheduler.c +++ b/src/or/scheduler.c @@ -13,8 +13,6 @@ #define TOR_CHANNEL_INTERNAL_ #include "channeltls.h" -#include <event2/event.h> - /** * \file scheduler.c * \brief Channel scheduling system: decides which channels should send and @@ -169,7 +167,7 @@ STATIC smartlist_t *channels_pending = NULL; * This event runs the scheduler from its callback, and is manually * activated whenever a channel enters open for writes/cells to send. */ -STATIC struct event *run_sched_ev = NULL; +STATIC struct mainloop_event_t *run_sched_ev = NULL; static int have_logged_kist_suddenly_disabled = 0; @@ -203,10 +201,9 @@ get_scheduler_type_string(scheduler_types_t type) * if any scheduling work was created during the event loop. */ static void -scheduler_evt_callback(evutil_socket_t fd, short events, void *arg) +scheduler_evt_callback(mainloop_event_t *event, void *arg) { - (void) fd; - (void) events; + (void) event; (void) arg; log_debug(LD_SCHED, "Scheduler event callback called"); @@ -487,10 +484,7 @@ scheduler_free_all(void) log_debug(LD_SCHED, "Shutting down scheduler"); if (run_sched_ev) { - if (event_del(run_sched_ev) < 0) { - log_warn(LD_BUG, "Problem deleting run_sched_ev"); - } - tor_event_free(run_sched_ev); + mainloop_event_free(run_sched_ev); run_sched_ev = NULL; } @@ -589,7 +583,7 @@ scheduler_ev_add(const struct timeval *next_run) { tor_assert(run_sched_ev); tor_assert(next_run); - if (BUG(event_add(run_sched_ev, next_run) < 0)) { + if (BUG(mainloop_event_schedule(run_sched_ev, next_run) < 0)) { log_warn(LD_SCHED, "Adding to libevent failed. Next run time was set to: " "%ld.%06ld", next_run->tv_sec, (long)next_run->tv_usec); return; @@ -598,10 +592,10 @@ scheduler_ev_add(const struct timeval *next_run) /** Make the scheduler event active with the given flags. */ void -scheduler_ev_active(int flags) +scheduler_ev_active(void) { tor_assert(run_sched_ev); - event_active(run_sched_ev, flags, 1); + mainloop_event_activate(run_sched_ev); } /* @@ -618,11 +612,10 @@ scheduler_init(void) IF_BUG_ONCE(!!run_sched_ev) { log_warn(LD_SCHED, "We should not already have a libevent scheduler event." "I'll clean the old one up, but this is odd."); - tor_event_free(run_sched_ev); + mainloop_event_free(run_sched_ev); run_sched_ev = NULL; } - run_sched_ev = tor_event_new(tor_libevent_get_base(), -1, - 0, scheduler_evt_callback, NULL); + run_sched_ev = mainloop_event_new(scheduler_evt_callback, NULL); channels_pending = smartlist_new(); set_scheduler(); diff --git a/src/or/scheduler.h b/src/or/scheduler.h index aeba9e2b75..08b02e286f 100644 --- a/src/or/scheduler.h +++ b/src/or/scheduler.h @@ -155,12 +155,12 @@ void scheduler_bug_occurred(const channel_t *chan); smartlist_t *get_channels_pending(void); MOCK_DECL(int, scheduler_compare_channels, (const void *c1_v, const void *c2_v)); -void scheduler_ev_active(int flags); +void scheduler_ev_active(void); void scheduler_ev_add(const struct timeval *next_run); #ifdef TOR_UNIT_TESTS extern smartlist_t *channels_pending; -extern struct event *run_sched_ev; +extern struct mainloop_event_t *run_sched_ev; extern const scheduler_t *the_scheduler; void scheduler_touch_channel(channel_t *chan); #endif /* defined(TOR_UNIT_TESTS) */ diff --git a/src/or/scheduler_kist.c b/src/or/scheduler_kist.c index 6d6490077d..c6e9b72c48 100644 --- a/src/or/scheduler_kist.c +++ b/src/or/scheduler_kist.c @@ -3,8 +3,6 @@ #define SCHEDULER_KIST_PRIVATE -#include <event2/event.h> - #include "or.h" #include "buffers.h" #include "config.h" @@ -553,7 +551,7 @@ kist_scheduler_schedule(void) /* Re-adding an event reschedules it. It does not duplicate it. */ scheduler_ev_add(&next_run); } else { - scheduler_ev_active(EV_TIMEOUT); + scheduler_ev_active(); } } diff --git a/src/or/scheduler_vanilla.c b/src/or/scheduler_vanilla.c index 7a83b9da18..b674d8256c 100644 --- a/src/or/scheduler_vanilla.c +++ b/src/or/scheduler_vanilla.c @@ -1,8 +1,6 @@ /* Copyright (c) 2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ -#include <event2/event.h> - #include "or.h" #include "config.h" #define TOR_CHANNEL_INTERNAL_ @@ -42,7 +40,7 @@ vanilla_scheduler_schedule(void) } /* Activate our event so it can process channels. */ - scheduler_ev_active(EV_TIMEOUT); + scheduler_ev_active(); } static void diff --git a/src/or/shared_random_client.c b/src/or/shared_random_client.c new file mode 100644 index 0000000000..3aef83cef4 --- /dev/null +++ b/src/or/shared_random_client.c @@ -0,0 +1,259 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file shared_random_client.c + * \brief This file contains functions that are from the shared random + * subsystem but used by many part of tor. The full feature is built + * as part of the dirauth module. + **/ + +#define SHARED_RANDOM_CLIENT_PRIVATE +#include "shared_random_client.h" + +#include "config.h" +#include "voting_schedule.h" +#include "networkstatus.h" +#include "util.h" +#include "util_format.h" + +/* Convert a given srv object to a string for the control port. This doesn't + * fail and the srv object MUST be valid. */ +static char * +srv_to_control_string(const sr_srv_t *srv) +{ + char *srv_str; + char srv_hash_encoded[SR_SRV_VALUE_BASE64_LEN + 1]; + tor_assert(srv); + + sr_srv_encode(srv_hash_encoded, sizeof(srv_hash_encoded), srv); + tor_asprintf(&srv_str, "%s", srv_hash_encoded); + return srv_str; +} + +/* Return the voting interval of the tor vote subsystem. */ +int +get_voting_interval(void) +{ + int interval; + networkstatus_t *consensus = networkstatus_get_live_consensus(time(NULL)); + + if (consensus) { + interval = (int)(consensus->fresh_until - consensus->valid_after); + } else { + /* Same for both a testing and real network. We voluntarily ignore the + * InitialVotingInterval since it complexifies things and it doesn't + * affect the SR protocol. */ + interval = get_options()->V3AuthVotingInterval; + } + tor_assert(interval > 0); + return interval; +} + +/* Given the time <b>now</b>, return the start time of the current round of + * the SR protocol. For example, if it's 23:47:08, the current round thus + * started at 23:47:00 for a voting interval of 10 seconds. */ +time_t +get_start_time_of_current_round(void) +{ + const or_options_t *options = get_options(); + int voting_interval = get_voting_interval(); + /* First, get the start time of the next round */ + time_t next_start = voting_schedule_get_next_valid_after_time(); + /* Now roll back next_start by a voting interval to find the start time of + the current round. */ + time_t curr_start = voting_schedule_get_start_of_next_interval( + next_start - voting_interval - 1, + voting_interval, + options->TestingV3AuthVotingStartOffset); + return curr_start; +} + +/* + * Public API + */ + +/* Encode the given shared random value and put it in dst. Destination + * buffer must be at least SR_SRV_VALUE_BASE64_LEN plus the NULL byte. */ +void +sr_srv_encode(char *dst, size_t dst_len, const sr_srv_t *srv) +{ + int ret; + /* Extra byte for the NULL terminated char. */ + char buf[SR_SRV_VALUE_BASE64_LEN + 1]; + + tor_assert(dst); + tor_assert(srv); + tor_assert(dst_len >= sizeof(buf)); + + ret = base64_encode(buf, sizeof(buf), (const char *) srv->value, + sizeof(srv->value), 0); + /* Always expect the full length without the NULL byte. */ + tor_assert(ret == (sizeof(buf) - 1)); + tor_assert(ret <= (int) dst_len); + strlcpy(dst, buf, dst_len); +} + +/* Return the current SRV string representation for the control port. Return a + * newly allocated string on success containing the value else "" if not found + * or if we don't have a valid consensus yet. */ +char * +sr_get_current_for_control(void) +{ + char *srv_str; + const networkstatus_t *c = networkstatus_get_latest_consensus(); + if (c && c->sr_info.current_srv) { + srv_str = srv_to_control_string(c->sr_info.current_srv); + } else { + srv_str = tor_strdup(""); + } + return srv_str; +} + +/* Return the previous SRV string representation for the control port. Return + * a newly allocated string on success containing the value else "" if not + * found or if we don't have a valid consensus yet. */ +char * +sr_get_previous_for_control(void) +{ + char *srv_str; + const networkstatus_t *c = networkstatus_get_latest_consensus(); + if (c && c->sr_info.previous_srv) { + srv_str = srv_to_control_string(c->sr_info.previous_srv); + } else { + srv_str = tor_strdup(""); + } + return srv_str; +} + +/* Return current shared random value from the latest consensus. Caller can + * NOT keep a reference to the returned pointer. Return NULL if none. */ +const sr_srv_t * +sr_get_current(const networkstatus_t *ns) +{ + const networkstatus_t *consensus; + + /* Use provided ns else get a live one */ + if (ns) { + consensus = ns; + } else { + consensus = networkstatus_get_live_consensus(approx_time()); + } + /* Ideally we would never be asked for an SRV without a live consensus. Make + * sure this assumption is correct. */ + tor_assert_nonfatal(consensus); + + if (consensus) { + return consensus->sr_info.current_srv; + } + return NULL; +} + +/* Return previous shared random value from the latest consensus. Caller can + * NOT keep a reference to the returned pointer. Return NULL if none. */ +const sr_srv_t * +sr_get_previous(const networkstatus_t *ns) +{ + const networkstatus_t *consensus; + + /* Use provided ns else get a live one */ + if (ns) { + consensus = ns; + } else { + consensus = networkstatus_get_live_consensus(approx_time()); + } + /* Ideally we would never be asked for an SRV without a live consensus. Make + * sure this assumption is correct. */ + tor_assert_nonfatal(consensus); + + if (consensus) { + return consensus->sr_info.previous_srv; + } + return NULL; +} + +/* Parse a list of arguments from a SRV value either from a vote, consensus + * or from our disk state and return a newly allocated srv object. NULL is + * returned on error. + * + * The arguments' order: + * num_reveals, value + */ +sr_srv_t * +sr_parse_srv(const smartlist_t *args) +{ + char *value; + int ok, ret; + uint64_t num_reveals; + sr_srv_t *srv = NULL; + + tor_assert(args); + + if (smartlist_len(args) < 2) { + goto end; + } + + /* First argument is the number of reveal values */ + num_reveals = tor_parse_uint64(smartlist_get(args, 0), + 10, 0, UINT64_MAX, &ok, NULL); + if (!ok) { + goto end; + } + /* Second and last argument is the shared random value it self. */ + value = smartlist_get(args, 1); + if (strlen(value) != SR_SRV_VALUE_BASE64_LEN) { + goto end; + } + + srv = tor_malloc_zero(sizeof(*srv)); + srv->num_reveals = num_reveals; + /* We subtract one byte from the srclen because the function ignores the + * '=' character in the given buffer. This is broken but it's a documented + * behavior of the implementation. */ + ret = base64_decode((char *) srv->value, sizeof(srv->value), value, + SR_SRV_VALUE_BASE64_LEN - 1); + if (ret != sizeof(srv->value)) { + tor_free(srv); + srv = NULL; + goto end; + } + end: + return srv; +} + +/** Return the start time of the current SR protocol run. For example, if the + * time is 23/06/2017 23:47:08 and a full SR protocol run is 24 hours, this + * function should return 23/06/2017 00:00:00. */ +time_t +sr_state_get_start_time_of_current_protocol_run(time_t now) +{ + int total_rounds = SHARED_RANDOM_N_ROUNDS * SHARED_RANDOM_N_PHASES; + int voting_interval = get_voting_interval(); + /* Find the time the current round started. */ + time_t beginning_of_current_round = get_start_time_of_current_round(); + + /* Get current SR protocol round */ + int current_round = (now / voting_interval) % total_rounds; + + /* Get start time by subtracting the time elapsed from the beginning of the + protocol run */ + time_t time_elapsed_since_start_of_run = current_round * voting_interval; + return beginning_of_current_round - time_elapsed_since_start_of_run; +} + +/** Return the time (in seconds) it takes to complete a full SR protocol phase + * (e.g. the commit phase). */ +unsigned int +sr_state_get_phase_duration(void) +{ + return SHARED_RANDOM_N_ROUNDS * get_voting_interval(); +} + +/** Return the time (in seconds) it takes to complete a full SR protocol run */ +unsigned int +sr_state_get_protocol_run_duration(void) +{ + int total_protocol_rounds = SHARED_RANDOM_N_ROUNDS * SHARED_RANDOM_N_PHASES; + return total_protocol_rounds * get_voting_interval(); +} + diff --git a/src/or/shared_random_client.h b/src/or/shared_random_client.h new file mode 100644 index 0000000000..89c608d45f --- /dev/null +++ b/src/or/shared_random_client.h @@ -0,0 +1,47 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file shared_random_client.h + * \brief Header file for shared_random_client.c. + **/ + +#ifndef TOR_SHARED_RANDOM_CLIENT_H +#define TOR_SHARED_RANDOM_CLIENT_H + +/* Dirauth module. */ +#include "dirauth/shared_random.h" + +/* Helper functions. */ +void sr_srv_encode(char *dst, size_t dst_len, const sr_srv_t *srv); +int get_voting_interval(void); + +/* Control port functions. */ +char *sr_get_current_for_control(void); +char *sr_get_previous_for_control(void); + +/* SRV functions. */ +const sr_srv_t *sr_get_current(const networkstatus_t *ns); +const sr_srv_t *sr_get_previous(const networkstatus_t *ns); +sr_srv_t *sr_parse_srv(const smartlist_t *args); + +/* + * Shared Random State API + */ + +/* Each protocol phase has 12 rounds */ +#define SHARED_RANDOM_N_ROUNDS 12 +/* Number of phase we have in a protocol. */ +#define SHARED_RANDOM_N_PHASES 2 + +time_t sr_state_get_start_time_of_current_protocol_run(time_t now); +unsigned int sr_state_get_phase_duration(void); +unsigned int sr_state_get_protocol_run_duration(void); +time_t get_start_time_of_current_round(void); + +#ifdef TOR_UNIT_TESTS + +#endif /* TOR_UNIT_TESTS */ + +#endif /* TOR_SHARED_RANDOM_CLIENT_H */ + diff --git a/src/or/statefile.c b/src/or/statefile.c index cc114f0a2b..c81ea44e06 100644 --- a/src/or/statefile.c +++ b/src/or/statefile.c @@ -37,6 +37,7 @@ #include "control.h" #include "entrynodes.h" #include "hibernate.h" +#include "main.h" #include "rephist.h" #include "router.h" #include "sandbox.h" @@ -680,6 +681,18 @@ save_transport_to_state(const char *transport, tor_free(transport_addrport); } +/** Change the next_write time of <b>state</b> to <b>when</b>, unless the + * state is already scheduled to be written to disk earlier than <b>when</b>. + */ +void +or_state_mark_dirty(or_state_t *state, time_t when) +{ + if (state->next_write > when) { + state->next_write = when; + reschedule_or_state_save(); + } +} + STATIC void or_state_free_(or_state_t *state) { diff --git a/src/or/statefile.h b/src/or/statefile.h index b4cc4d1dc6..5aa2ca9320 100644 --- a/src/or/statefile.h +++ b/src/or/statefile.h @@ -17,6 +17,7 @@ char *get_stored_bindaddr_for_server_transport(const char *transport); int or_state_load(void); int or_state_loaded(void); void or_state_free_all(void); +void or_state_mark_dirty(or_state_t *state, time_t when); #ifdef STATEFILE_PRIVATE STATIC config_line_t *get_transport_in_state_by_name(const char *transport); diff --git a/src/or/status.c b/src/or/status.c index 4f7be164b1..4b8033d114 100644 --- a/src/or/status.c +++ b/src/or/status.c @@ -25,7 +25,6 @@ #include "main.h" #include "rephist.h" #include "hibernate.h" -#include "rephist.h" #include "statefile.h" #include "hs_stats.h" #include "hs_service.h" @@ -88,19 +87,19 @@ bytes_to_usage(uint64_t bytes) return bw_string; } -/** Log some usage info about our hidden service */ +/** Log some usage info about our onion service(s). */ static void log_onion_service_stats(void) { unsigned int num_services = hs_service_get_num_services(); - /* If there are no active hidden services, no need to print logs */ + /* If there are no active onion services, no need to print logs */ if (num_services == 0) { return; } log_notice(LD_HEARTBEAT, - "Our hidden service%s received %u v2 and %u v3 INTRODUCE2 cells " + "Our onion service%s received %u v2 and %u v3 INTRODUCE2 cells " "and attempted to launch %d rendezvous circuits.", num_services == 1 ? "" : "s", hs_stats_get_n_introduce2_v2_cells(), diff --git a/src/or/torcert.c b/src/or/torcert.c index 51935ddf72..1c5afd965a 100644 --- a/src/or/torcert.c +++ b/src/or/torcert.c @@ -27,7 +27,7 @@ #include "or.h" #include "config.h" -#include "crypto.h" +#include "crypto_util.h" #include "torcert.h" #include "ed25519_cert.h" #include "torlog.h" diff --git a/src/or/transports.c b/src/or/transports.c index b08dcd1613..614fc81da8 100644 --- a/src/or/transports.c +++ b/src/or/transports.c @@ -135,7 +135,7 @@ static smartlist_t *transport_list = NULL; /** Returns a transport_t struct for a transport proxy supporting the protocol <b>name</b> listening at <b>addr</b>:<b>port</b> using SOCKS version <b>socks_ver</b>. */ -static transport_t * +STATIC transport_t * transport_new(const tor_addr_t *addr, uint16_t port, const char *name, int socks_ver, const char *extra_info_args) @@ -222,8 +222,8 @@ transport_copy(const transport_t *transport) /** Returns the transport in our transport list that has the name <b>name</b>. * Else returns NULL. */ -transport_t * -transport_get_by_name(const char *name) +MOCK_IMPL(transport_t *, +transport_get_by_name,(const char *name)) { tor_assert(name); @@ -1025,48 +1025,71 @@ parse_method_error(const char *line, int is_server) line+strlen(error)+1); } -/** Parses an SMETHOD <b>line</b> and if well-formed it registers the - * new transport in <b>mp</b>. */ -STATIC int -parse_smethod_line(const char *line, managed_proxy_t *mp) +/** A helper for parse_{c,s}method_line(), bootstraps its + * functionalities. If <b>is_smethod</b> is true then the + * the line to parse is a SMETHOD line otherwise it is a + * CMETHOD line*/ +static int +parse_method_line_helper(const char *line, + managed_proxy_t *mp, + int is_smethod) { + int item_index = 0; int r; - smartlist_t *items = NULL; - char *method_name=NULL; + char *transport_name=NULL; char *args_string=NULL; char *addrport=NULL; - tor_addr_t tor_addr; + int socks_ver=PROXY_NONE; char *address=NULL; uint16_t port = 0; + const char *method_str = is_smethod ? PROTO_SMETHOD : PROTO_CMETHOD; + const int min_args_count = is_smethod ? 3 : 4; + + tor_addr_t tor_addr; transport_t *transport=NULL; + smartlist_t *items= smartlist_new(); - items = smartlist_new(); smartlist_split_string(items, line, NULL, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1); - if (smartlist_len(items) < 3) { - log_warn(LD_CONFIG, "Server managed proxy sent us a SMETHOD line " - "with too few arguments."); + if (smartlist_len(items) < min_args_count) { + log_warn(LD_CONFIG, "Managed proxy sent us a %s line " + "with too few arguments.", method_str); goto err; } - /* Example of legit SMETHOD line: - SMETHOD obfs2 0.0.0.0:25612 ARGS:secret=supersekrit,key=superkey */ - - tor_assert(!strcmp(smartlist_get(items,0),PROTO_SMETHOD)); + tor_assert(!strcmp(smartlist_get(items, item_index),method_str)); + ++item_index; - method_name = smartlist_get(items,1); - if (!string_is_C_identifier(method_name)) { + transport_name = smartlist_get(items,item_index); + ++item_index; + if (!string_is_C_identifier(transport_name)) { log_warn(LD_CONFIG, "Transport name is not a C identifier (%s).", - method_name); + transport_name); goto err; } - addrport = smartlist_get(items, 2); + /** Check for the proxy method sent to us in CMETHOD line. */ + if (!is_smethod) { + const char *socks_ver_str = smartlist_get(items,item_index); + ++item_index; + + if (!strcmp(socks_ver_str,"socks4")) { + socks_ver = PROXY_SOCKS4; + } else if (!strcmp(socks_ver_str,"socks5")) { + socks_ver = PROXY_SOCKS5; + } else { + log_warn(LD_CONFIG, "Client managed proxy sent us a proxy protocol " + "we don't recognize. (%s)", socks_ver_str); + goto err; + } + } + + addrport = smartlist_get(items, item_index); + ++item_index; if (tor_addr_port_split(LOG_WARN, addrport, &address, &port)<0) { - log_warn(LD_CONFIG, "Error parsing transport " - "address '%s'", addrport); + log_warn(LD_CONFIG, "Error parsing transport address '%s'", addrport); goto err; } @@ -1081,10 +1104,11 @@ parse_smethod_line(const char *line, managed_proxy_t *mp) goto err; } - if (smartlist_len(items) > 3) { + /** Check for options in the SMETHOD line. */ + if (is_smethod && smartlist_len(items) > min_args_count) { /* Seems like there are also some [options] in the SMETHOD line. Let's see if we can parse them. */ - char *options_string = smartlist_get(items, 3); + char *options_string = smartlist_get(items, item_index); log_debug(LD_CONFIG, "Got options_string: %s", options_string); if (!strcmpstart(options_string, "ARGS:")) { args_string = options_string+strlen("ARGS:"); @@ -1092,15 +1116,20 @@ parse_smethod_line(const char *line, managed_proxy_t *mp) } } - transport = transport_new(&tor_addr, port, method_name, - PROXY_NONE, args_string); + transport = transport_new(&tor_addr, port, transport_name, + socks_ver, args_string); smartlist_add(mp->transports, transport); - /* For now, notify the user so that they know where the server - transport is listening. */ - log_info(LD_CONFIG, "Server transport %s at %s:%d.", - method_name, address, (int)port); + /** Logs info about line parsing success for client or server */ + if (is_smethod) { + log_info(LD_CONFIG, "Server transport %s at %s:%d.", + transport_name, address, (int)port); + } else { + log_info(LD_CONFIG, "Transport %s at %s:%d with SOCKS %d. " + "Attached to managed proxy.", + transport_name, address, (int)port, socks_ver); + } r=0; goto done; @@ -1115,93 +1144,24 @@ parse_smethod_line(const char *line, managed_proxy_t *mp) return r; } +/** Parses an SMETHOD <b>line</b> and if well-formed it registers the + * new transport in <b>mp</b>. */ +STATIC int +parse_smethod_line(const char *line, managed_proxy_t *mp) +{ + /* Example of legit SMETHOD line: + SMETHOD obfs2 0.0.0.0:25612 ARGS:secret=supersekrit,key=superkey */ + return parse_method_line_helper(line, mp, 1); +} + /** Parses a CMETHOD <b>line</b>, and if well-formed it registers * the new transport in <b>mp</b>. */ STATIC int parse_cmethod_line(const char *line, managed_proxy_t *mp) { - int r; - smartlist_t *items = NULL; - - char *method_name=NULL; - - char *socks_ver_str=NULL; - int socks_ver=PROXY_NONE; - - char *addrport=NULL; - tor_addr_t tor_addr; - char *address=NULL; - uint16_t port = 0; - - transport_t *transport=NULL; - - items = smartlist_new(); - smartlist_split_string(items, line, NULL, - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1); - if (smartlist_len(items) < 4) { - log_warn(LD_CONFIG, "Client managed proxy sent us a CMETHOD line " - "with too few arguments."); - goto err; - } - - tor_assert(!strcmp(smartlist_get(items,0),PROTO_CMETHOD)); - - method_name = smartlist_get(items,1); - if (!string_is_C_identifier(method_name)) { - log_warn(LD_CONFIG, "Transport name is not a C identifier (%s).", - method_name); - goto err; - } - - socks_ver_str = smartlist_get(items,2); - - if (!strcmp(socks_ver_str,"socks4")) { - socks_ver = PROXY_SOCKS4; - } else if (!strcmp(socks_ver_str,"socks5")) { - socks_ver = PROXY_SOCKS5; - } else { - log_warn(LD_CONFIG, "Client managed proxy sent us a proxy protocol " - "we don't recognize. (%s)", socks_ver_str); - goto err; - } - - addrport = smartlist_get(items, 3); - if (tor_addr_port_split(LOG_WARN, addrport, &address, &port)<0) { - log_warn(LD_CONFIG, "Error parsing transport " - "address '%s'", addrport); - goto err; - } - - if (!port) { - log_warn(LD_CONFIG, - "Transport address '%s' has no port.", addrport); - goto err; - } - - if (tor_addr_parse(&tor_addr, address) < 0) { - log_warn(LD_CONFIG, "Error parsing transport address '%s'", address); - goto err; - } - - transport = transport_new(&tor_addr, port, method_name, socks_ver, NULL); - - smartlist_add(mp->transports, transport); - - log_info(LD_CONFIG, "Transport %s at %s:%d with SOCKS %d. " - "Attached to managed proxy.", - method_name, address, (int)port, socks_ver); - - r=0; - goto done; - - err: - r = -1; - - done: - SMARTLIST_FOREACH(items, char*, s, tor_free(s)); - smartlist_free(items); - tor_free(address); - return r; + /* Example of legit CMETHOD line: + CMETHOD obfs2 socks5 127.0.0.1:35713 */ + return parse_method_line_helper(line, mp, 0); } /** Parses an PROXY-ERROR <b>line</b> and warns the user accordingly. */ diff --git a/src/or/transports.h b/src/or/transports.h index 1b2786472c..022b926a03 100644 --- a/src/or/transports.h +++ b/src/or/transports.h @@ -38,7 +38,7 @@ MOCK_DECL(int, transport_add_from_config, void transport_free_(transport_t *transport); #define transport_free(tr) FREE_AND_NULL(transport_t, transport_free_, (tr)) -transport_t *transport_get_by_name(const char *name); +MOCK_DECL(transport_t*, transport_get_by_name, (const char *name)); MOCK_DECL(void, pt_kickstart_proxy, (const smartlist_t *transport_list, char **proxy_argv, @@ -113,6 +113,9 @@ typedef struct { smartlist_t *transports; } managed_proxy_t; +STATIC transport_t *transport_new(const tor_addr_t *addr, uint16_t port, + const char *name, int socks_ver, + const char *extra_info_args); STATIC int parse_cmethod_line(const char *line, managed_proxy_t *mp); STATIC int parse_smethod_line(const char *line, managed_proxy_t *mp); diff --git a/src/or/voting_schedule.c b/src/or/voting_schedule.c new file mode 100644 index 0000000000..d230a6dbcd --- /dev/null +++ b/src/or/voting_schedule.c @@ -0,0 +1,192 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file voting_schedule.c + * \brief This file contains functions that are from the directory authority + * subsystem related to voting specifically but used by many part of + * tor. The full feature is built as part of the dirauth module. + **/ + +#define VOTING_SCHEDULE_PRIVATE +#include "voting_schedule.h" + +#include "or.h" +#include "config.h" +#include "networkstatus.h" + +/* ===== + * Vote scheduling + * ===== */ + +/** Return the start of the next interval of size <b>interval</b> (in + * seconds) after <b>now</b>, plus <b>offset</b>. Midnight always + * starts a fresh interval, and if the last interval of a day would be + * truncated to less than half its size, it is rolled into the + * previous interval. */ +time_t +voting_schedule_get_start_of_next_interval(time_t now, int interval, + int offset) +{ + struct tm tm; + time_t midnight_today=0; + time_t midnight_tomorrow; + time_t next; + + tor_gmtime_r(&now, &tm); + tm.tm_hour = 0; + tm.tm_min = 0; + tm.tm_sec = 0; + + if (tor_timegm(&tm, &midnight_today) < 0) { + // LCOV_EXCL_START + log_warn(LD_BUG, "Ran into an invalid time when trying to find midnight."); + // LCOV_EXCL_STOP + } + midnight_tomorrow = midnight_today + (24*60*60); + + next = midnight_today + ((now-midnight_today)/interval + 1)*interval; + + /* Intervals never cross midnight. */ + if (next > midnight_tomorrow) + next = midnight_tomorrow; + + /* If the interval would only last half as long as it's supposed to, then + * skip over to the next day. */ + if (next + interval/2 > midnight_tomorrow) + next = midnight_tomorrow; + + next += offset; + if (next - interval > now) + next -= interval; + + return next; +} + +/* Populate and return a new voting_schedule_t that can be used to schedule + * voting. The object is allocated on the heap and it's the responsibility of + * the caller to free it. Can't fail. */ +static voting_schedule_t * +get_voting_schedule(const or_options_t *options, time_t now, int severity) +{ + int interval, vote_delay, dist_delay; + time_t start; + time_t end; + networkstatus_t *consensus; + voting_schedule_t *new_voting_schedule; + + new_voting_schedule = tor_malloc_zero(sizeof(voting_schedule_t)); + + consensus = networkstatus_get_live_consensus(now); + + if (consensus) { + interval = (int)( consensus->fresh_until - consensus->valid_after ); + vote_delay = consensus->vote_seconds; + dist_delay = consensus->dist_seconds; + + /* Note down the consensus valid after, so that we detect outdated voting + * schedules in case of skewed clocks etc. */ + new_voting_schedule->live_consensus_valid_after = consensus->valid_after; + } else { + interval = options->TestingV3AuthInitialVotingInterval; + vote_delay = options->TestingV3AuthInitialVoteDelay; + dist_delay = options->TestingV3AuthInitialDistDelay; + } + + tor_assert(interval > 0); + + if (vote_delay + dist_delay > interval/2) + vote_delay = dist_delay = interval / 4; + + start = new_voting_schedule->interval_starts = + voting_schedule_get_start_of_next_interval(now,interval, + options->TestingV3AuthVotingStartOffset); + end = voting_schedule_get_start_of_next_interval(start+1, interval, + options->TestingV3AuthVotingStartOffset); + + tor_assert(end > start); + + new_voting_schedule->fetch_missing_signatures = start - (dist_delay/2); + new_voting_schedule->voting_ends = start - dist_delay; + new_voting_schedule->fetch_missing_votes = + start - dist_delay - (vote_delay/2); + new_voting_schedule->voting_starts = start - dist_delay - vote_delay; + + { + char tbuf[ISO_TIME_LEN+1]; + format_iso_time(tbuf, new_voting_schedule->interval_starts); + tor_log(severity, LD_DIR,"Choosing expected valid-after time as %s: " + "consensus_set=%d, interval=%d", + tbuf, consensus?1:0, interval); + } + + return new_voting_schedule; +} + +#define voting_schedule_free(s) \ + FREE_AND_NULL(voting_schedule_t, voting_schedule_free_, (s)) + +/** Frees a voting_schedule_t. This should be used instead of the generic + * tor_free. */ +static void +voting_schedule_free_(voting_schedule_t *voting_schedule_to_free) +{ + if (!voting_schedule_to_free) + return; + tor_free(voting_schedule_to_free); +} + +voting_schedule_t voting_schedule; + +/* Using the time <b>now</b>, return the next voting valid-after time. */ +time_t +voting_schedule_get_next_valid_after_time(void) +{ + time_t now = approx_time(); + bool need_to_recalculate_voting_schedule = false; + + /* This is a safe guard in order to make sure that the voting schedule + * static object is at least initialized. Using this function with a zeroed + * voting schedule can lead to bugs. */ + if (tor_mem_is_zero((const char *) &voting_schedule, + sizeof(voting_schedule))) { + need_to_recalculate_voting_schedule = true; + goto done; /* no need for next check if we have to recalculate anyway */ + } + + /* Also make sure we are not using an outdated voting schedule. If we have a + * newer consensus, make sure we recalculate the voting schedule. */ + const networkstatus_t *ns = networkstatus_get_live_consensus(now); + if (ns && ns->valid_after != voting_schedule.live_consensus_valid_after) { + log_info(LD_DIR, "Voting schedule is outdated: recalculating (%d/%d)", + (int) ns->valid_after, + (int) voting_schedule.live_consensus_valid_after); + need_to_recalculate_voting_schedule = true; + } + + done: + if (need_to_recalculate_voting_schedule) { + voting_schedule_recalculate_timing(get_options(), now); + voting_schedule.created_on_demand = 1; + } + + return voting_schedule.interval_starts; +} + +/** Set voting_schedule to hold the timing for the next vote we should be + * doing. All type of tor do that because HS subsystem needs the timing as + * well to function properly. */ +void +voting_schedule_recalculate_timing(const or_options_t *options, time_t now) +{ + voting_schedule_t *new_voting_schedule; + + /* get the new voting schedule */ + new_voting_schedule = get_voting_schedule(options, now, LOG_INFO); + tor_assert(new_voting_schedule); + + /* Fill in the global static struct now */ + memcpy(&voting_schedule, new_voting_schedule, sizeof(voting_schedule)); + voting_schedule_free(new_voting_schedule); +} + diff --git a/src/or/voting_schedule.h b/src/or/voting_schedule.h new file mode 100644 index 0000000000..087701408e --- /dev/null +++ b/src/or/voting_schedule.h @@ -0,0 +1,65 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file voting_schedule.h + * \brief Header file for voting_schedule.c. + **/ + +#ifndef TOR_VOTING_SCHEDULE_H +#define TOR_VOTING_SCHEDULE_H + +#include "or.h" + +/** Scheduling information for a voting interval. */ +typedef struct { + /** When do we generate and distribute our vote for this interval? */ + time_t voting_starts; + /** When do we send an HTTP request for any votes that we haven't + * been posted yet?*/ + time_t fetch_missing_votes; + /** When do we give up on getting more votes and generate a consensus? */ + time_t voting_ends; + /** When do we send an HTTP request for any signatures we're expecting to + * see on the consensus? */ + time_t fetch_missing_signatures; + /** When do we publish the consensus? */ + time_t interval_starts; + + /* True iff we have generated and distributed our vote. */ + int have_voted; + /* True iff we've requested missing votes. */ + int have_fetched_missing_votes; + /* True iff we have built a consensus and sent the signatures around. */ + int have_built_consensus; + /* True iff we've fetched missing signatures. */ + int have_fetched_missing_signatures; + /* True iff we have published our consensus. */ + int have_published_consensus; + + /* True iff this voting schedule was set on demand meaning not through the + * normal vote operation of a dirauth or when a consensus is set. This only + * applies to a directory authority that needs to recalculate the voting + * timings only for the first vote even though this object was initilized + * prior to voting. */ + int created_on_demand; + + /** The valid-after time of the last live consensus that filled this voting + * schedule. It's used to detect outdated voting schedules. */ + time_t live_consensus_valid_after; +} voting_schedule_t; + +/* Public API. */ + +extern voting_schedule_t voting_schedule; + +void voting_schedule_recalculate_timing(const or_options_t *options, + time_t now); + +time_t voting_schedule_get_start_of_next_interval(time_t now, + int interval, + int offset); +time_t voting_schedule_get_next_valid_after_time(void); + +#endif /* TOR_VOTING_SCHEDULE_H */ + diff --git a/src/rust/.cargo/config.in b/src/rust/.cargo/config.in index 301e7fdbe7..6eddc75459 100644 --- a/src/rust/.cargo/config.in +++ b/src/rust/.cargo/config.in @@ -6,3 +6,7 @@ @RUST_DL@ [source.vendored-sources] @RUST_DL@ directory = '@TOR_RUST_DEPENDENCIES@' + +[build] +@RUST_WARN@ rustflags = [ "-D", "warnings" ] +@RUST_TARGET_PROP@ diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock index 4f918c0221..1d2a7359aa 100644 --- a/src/rust/Cargo.lock +++ b/src/rust/Cargo.lock @@ -1,8 +1,39 @@ [[package]] +name = "crypto" +version = "0.0.1" +dependencies = [ + "digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "external 0.0.1", + "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.5.0-pre.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.2.0-pre.0 (registry+https://github.com/rust-lang/crates.io-index)", + "smartlist 0.0.1", + "tor_allocate 0.0.1", + "tor_log 0.1.0", +] + +[[package]] +name = "digest" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] name = "external" version = "0.0.1" dependencies = [ "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", + "smartlist 0.0.1", +] + +[[package]] +name = "generic-array" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "typenum 1.9.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -18,10 +49,24 @@ dependencies = [ "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", "smartlist 0.0.1", "tor_allocate 0.0.1", + "tor_log 0.1.0", "tor_util 0.0.1", ] [[package]] +name = "rand" +version = "0.5.0-pre.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.2.0-pre.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_core" +version = "0.2.0-pre.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] name = "smartlist" version = "0.0.1" dependencies = [ @@ -36,6 +81,14 @@ dependencies = [ ] [[package]] +name = "tor_log" +version = "0.1.0" +dependencies = [ + "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", + "tor_allocate 0.0.1", +] + +[[package]] name = "tor_rust" version = "0.1.0" dependencies = [ @@ -49,7 +102,18 @@ version = "0.0.1" dependencies = [ "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", "tor_allocate 0.0.1", + "tor_log 0.1.0", ] +[[package]] +name = "typenum" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [metadata] +"checksum digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "00a49051fef47a72c9623101b19bd71924a45cca838826caae3eaa4d00772603" +"checksum generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ef25c5683767570c2bbd7deba372926a55eaae9982d7726ee2a1050239d45b9d" "checksum libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)" = "f54263ad99207254cf58b5f701ecb432c717445ea2ee8af387334bdd1a03fdff" +"checksum rand 0.5.0-pre.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3795e4701d9628a63a84d0289e66279883b40df165fca7caed7b87122447032a" +"checksum rand_core 0.2.0-pre.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c7255ffbdb188d5be1a69b6f9f3cf187de4207430b9e79ed5b76458a6b20de9a" +"checksum typenum 1.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "13a99dc6780ef33c78780b826cf9d2a78840b72cae9474de4bcaf9051e60ebbd" diff --git a/src/rust/Cargo.toml b/src/rust/Cargo.toml index 953c9b96b7..c3e44d2a79 100644 --- a/src/rust/Cargo.toml +++ b/src/rust/Cargo.toml @@ -1,5 +1,14 @@ [workspace] -members = ["tor_util", "protover", "smartlist", "external", "tor_allocate", "tor_rust"] +members = [ + "crypto", + "external", + "protover", + "smartlist", + "tor_allocate", + "tor_log", + "tor_rust", + "tor_util", +] [profile.release] debug = true diff --git a/src/rust/build.rs b/src/rust/build.rs new file mode 100644 index 0000000000..b943aa5535 --- /dev/null +++ b/src/rust/build.rs @@ -0,0 +1,179 @@ +//! Build script for Rust modules in Tor. +//! +//! We need to use this because some of our Rust tests need to use some +//! of our C modules, which need to link some external libraries. +//! +//! This script works by looking at a "config.rust" file generated by our +//! configure script, and then building a set of options for cargo to pass to +//! the compiler. + +use std::collections::HashMap; +use std::env; +use std::fs::File; +use std::io::prelude::*; +use std::io; +use std::path::PathBuf; + +/// Wrapper around a key-value map. +struct Config( + HashMap<String,String> +); + +/// Locate a config.rust file generated by autoconf, starting in the OUT_DIR +/// location provided by cargo and recursing up the directory tree. Note that +/// we need to look in the OUT_DIR, since autoconf will place generated files +/// in the build directory. +fn find_cfg() -> io::Result<String> { + let mut path = PathBuf::from(env::var("OUT_DIR").unwrap()); + loop { + path.push("config.rust"); + if path.exists() { + return Ok(path.to_str().unwrap().to_owned()); + } + path.pop(); // remove config.rust + if ! path.pop() { // can't remove last part of directory + return Err(io::Error::new(io::ErrorKind::NotFound, + "No config.rust")); + } + } +} + +impl Config { + /// Find the config.rust file and try to parse it. + /// + /// The file format is a series of lines of the form KEY=VAL, with + /// any blank lines and lines starting with # ignored. + fn load() -> io::Result<Config> { + let path = find_cfg()?; + let f = File::open(&path)?; + let reader = io::BufReader::new(f); + let mut map = HashMap::new(); + for line in reader.lines() { + let s = line?; + if s.trim().starts_with("#") || s.trim() == "" { + continue; + } + let idx = match s.find("=") { + None => { + return Err(io::Error::new(io::ErrorKind::InvalidData, + "missing =")); + }, + Some(x) => x + }; + let (var,eq_val) = s.split_at(idx); + let val = &eq_val[1..]; + map.insert(var.to_owned(), val.to_owned()); + } + Ok(Config(map)) + } + + /// Return a reference to the value whose key is 'key'. + /// + /// Panics if 'key' is not found in the configuration. + fn get(&self, key : &str) -> &str { + self.0.get(key).unwrap() + } + + /// Add a dependency on a static C library that is part of Tor, by name. + fn component(&self, s : &str) { + println!("cargo:rustc-link-lib=static={}", s); + } + + /// Add a dependency on a native library that is not part of Tor, by name. + fn dependency(&self, s : &str) { + println!("cargo:rustc-link-lib={}", s); + } + + /// Add a link path, relative to Tor's build directory. + fn link_relpath(&self, s : &str) { + let builddir = self.get("BUILDDIR"); + println!("cargo:rustc-link-search=native={}/{}", builddir, s); + } + + /// Add an absolute link path. + fn link_path(&self, s : &str) { + println!("cargo:rustc-link-search=native={}", s); + } + + /// Parse the CFLAGS in s, looking for -l and -L items, and adding + /// rust configuration as appropriate. + fn from_cflags(&self, s : &str) { + let mut next_is_lib = false; + let mut next_is_path = false; + for ent in self.get(s).split_whitespace() { + if next_is_lib { + self.dependency(ent); + next_is_lib = false; + } else if next_is_path { + self.link_path(ent); + next_is_path = false; + } else if ent == "-l" { + next_is_lib = true; + } else if ent == "-L" { + next_is_path = true; + } else if ent.starts_with("-L") { + self.link_path(&ent[2..]); + } else if ent.starts_with("-l") { + self.dependency(&ent[2..]); + } + } + } +} + +pub fn main() { + let cfg = Config::load().unwrap(); + let package = env::var("CARGO_PKG_NAME").unwrap(); + + match package.as_ref() { + "crypto" => { + // Right now, I'm having a separate configuration for each Rust + // package, since I'm hoping we can trim them down. Once we have a + // second Rust package that needs to use this build script, let's + // extract some of this stuff into a module. + // + // This is a ridiculous amount of code to be pulling in just + // to test our crypto library: modularity would be our + // friend here. + cfg.from_cflags("TOR_LDFLAGS_zlib"); + cfg.from_cflags("TOR_LDFLAGS_openssl"); + cfg.from_cflags("TOR_LDFLAGS_libevent"); + + cfg.link_relpath("src/common"); + cfg.link_relpath("src/ext/keccak-tiny"); + cfg.link_relpath("src/ext/keccak-tiny"); + cfg.link_relpath("src/ext/ed25519/ref10"); + cfg.link_relpath("src/ext/ed25519/donna"); + cfg.link_relpath("src/trunnel"); + + // Note that we can't pull in "libtor-testing", or else we + // will have dependencies on all the other rust packages that + // tor uses. We must be careful with factoring and dependencies + // moving forward! + cfg.component("or-crypto-testing"); + cfg.component("or-ctime-testing"); + cfg.component("or-testing"); + cfg.component("or-event-testing"); + cfg.component("or-ctime-testing"); + cfg.component("curve25519_donna"); + cfg.component("keccak-tiny"); + cfg.component("ed25519_ref10"); + cfg.component("ed25519_donna"); + cfg.component("or-trunnel-testing"); + + cfg.from_cflags("TOR_ZLIB_LIBS"); + cfg.from_cflags("TOR_LIB_MATH"); + cfg.from_cflags("TOR_OPENSSL_LIBS"); + cfg.from_cflags("TOR_LIBEVENT_LIBS"); + cfg.from_cflags("TOR_LIB_WS32"); + cfg.from_cflags("TOR_LIB_GDI"); + cfg.from_cflags("TOR_LIB_USERENV"); + cfg.from_cflags("CURVE25519_LIBS"); + cfg.from_cflags("TOR_LZMA_LIBS"); + cfg.from_cflags("TOR_ZSTD_LIBS"); + cfg.from_cflags("LIBS"); + }, + _ => { + panic!("No configuration in build.rs for package {}", package); + } + } +} diff --git a/src/rust/crypto/Cargo.toml b/src/rust/crypto/Cargo.toml new file mode 100644 index 0000000000..869e0d6256 --- /dev/null +++ b/src/rust/crypto/Cargo.toml @@ -0,0 +1,28 @@ +[package] +authors = ["The Tor Project", + "Isis Lovecruft <isis@torproject.org>"] +name = "crypto" +version = "0.0.1" +publish = false +build = "../build.rs" + +[lib] +name = "crypto" +path = "lib.rs" +crate_type = ["rlib", "staticlib"] + +[dependencies] +libc = "=0.2.39" +digest = "=0.7.2" +rand_core = { version = "=0.2.0-pre.0", default-features = false } + +external = { path = "../external" } +smartlist = { path = "../smartlist" } +tor_allocate = { path = "../tor_allocate" } +tor_log = { path = "../tor_log" } + +[dev-dependencies] +rand = { version = "=0.5.0-pre.2", default-features = false } +rand_core = { version = "=0.2.0-pre.0", default-features = false } + +[features] diff --git a/src/rust/crypto/digests/mod.rs b/src/rust/crypto/digests/mod.rs new file mode 100644 index 0000000000..a2463b89eb --- /dev/null +++ b/src/rust/crypto/digests/mod.rs @@ -0,0 +1,7 @@ +// Copyright (c) 2018, The Tor Project, Inc. +// Copyright (c) 2018, isis agora lovecruft +// See LICENSE for licensing information + +//! Hash Digests and eXtendible Output Functions (XOFs) + +pub mod sha2; diff --git a/src/rust/crypto/digests/sha2.rs b/src/rust/crypto/digests/sha2.rs new file mode 100644 index 0000000000..03e0843dc0 --- /dev/null +++ b/src/rust/crypto/digests/sha2.rs @@ -0,0 +1,222 @@ +// Copyright (c) 2018, The Tor Project, Inc. +// Copyright (c) 2018, isis agora lovecruft +// See LICENSE for licensing information + +//! Hash Digests and eXtendible Output Functions (XOFs) + +pub use digest::Digest; + +use digest::BlockInput; +use digest::FixedOutput; +use digest::Input; +use digest::generic_array::GenericArray; +use digest::generic_array::typenum::U32; +use digest::generic_array::typenum::U64; + +use external::crypto_digest::CryptoDigest; +use external::crypto_digest::DigestAlgorithm; +use external::crypto_digest::get_256_bit_digest; +use external::crypto_digest::get_512_bit_digest; + +pub use external::crypto_digest::DIGEST256_LEN; +pub use external::crypto_digest::DIGEST512_LEN; + +/// The block size for both SHA-256 and SHA-512 digests is 512 bits/64 bytes. +/// +/// Unfortunately, we have to use the generic_array crate currently to express +/// this at compile time. Later, in the future, when Rust implements const +/// generics, we'll be able to remove this dependency (actually, it will get +/// removed from the digest crate, which is currently `pub use`ing it). +type BlockSize = U64; + +/// A SHA2-256 digest. +/// +/// # C_RUST_COUPLED +/// +/// * `crypto_digest_dup` +#[derive(Clone)] +pub struct Sha256 { + engine: CryptoDigest, +} + +/// Construct a new, default instance of a `Sha256` hash digest function. +/// +/// # Examples +/// +/// ```rust,no_run +/// use crypto::digests::sha2::{Sha256, Digest}; +/// +/// let mut hasher: Sha256 = Sha256::default(); +/// ``` +/// +/// # Returns +/// +/// A new `Sha256` digest. +impl Default for Sha256 { + fn default() -> Sha256 { + Sha256{ engine: CryptoDigest::new(Some(DigestAlgorithm::SHA2_256)) } + } +} + +impl BlockInput for Sha256 { + type BlockSize = BlockSize; +} + +/// Input `msg` into the digest. +/// +/// # Examples +/// +/// ```rust,no_run +/// use crypto::digests::sha2::{Sha256, Digest}; +/// +/// let mut hasher: Sha256 = Sha256::default(); +/// +/// hasher.input(b"foo"); +/// hasher.input(b"bar"); +/// ``` +impl Input for Sha256 { + fn process(&mut self, msg: &[u8]) { + self.engine.add_bytes(&msg); + } +} + +/// Retrieve the output hash from everything which has been fed into this +/// `Sha256` digest thus far. +/// +// +// FIXME: Once const generics land in Rust, we should genericise calling +// crypto_digest_get_digest in external::crypto_digest. +impl FixedOutput for Sha256 { + type OutputSize = U32; + + fn fixed_result(self) -> GenericArray<u8, Self::OutputSize> { + let buffer: [u8; DIGEST256_LEN] = get_256_bit_digest(self.engine); + + GenericArray::from(buffer) + } +} + +/// A SHA2-512 digest. +/// +/// # C_RUST_COUPLED +/// +/// * `crypto_digest_dup` +#[derive(Clone)] +pub struct Sha512 { + engine: CryptoDigest, +} + +/// Construct a new, default instance of a `Sha512` hash digest function. +/// +/// # Examples +/// +/// ```rust,no_run +/// use crypto::digests::sha2::{Sha512, Digest}; +/// +/// let mut hasher: Sha512 = Sha512::default(); +/// ``` +/// +/// # Returns +/// +/// A new `Sha512` digest. +impl Default for Sha512 { + fn default() -> Sha512 { + Sha512{ engine: CryptoDigest::new(Some(DigestAlgorithm::SHA2_512)) } + } +} + +impl BlockInput for Sha512 { + type BlockSize = BlockSize; +} + +/// Input `msg` into the digest. +/// +/// # Examples +/// +/// ```rust,no_run +/// use crypto::digests::sha2::{Sha512, Digest}; +/// +/// let mut hasher: Sha512 = Sha512::default(); +/// +/// hasher.input(b"foo"); +/// hasher.input(b"bar"); +/// ``` +impl Input for Sha512 { + fn process(&mut self, msg: &[u8]) { + self.engine.add_bytes(&msg); + } +} + +/// Retrieve the output hash from everything which has been fed into this +/// `Sha512` digest thus far. +/// +// +// FIXME: Once const generics land in Rust, we should genericise calling +// crypto_digest_get_digest in external::crypto_digest. +impl FixedOutput for Sha512 { + type OutputSize = U64; + + fn fixed_result(self) -> GenericArray<u8, Self::OutputSize> { + let buffer: [u8; DIGEST512_LEN] = get_512_bit_digest(self.engine); + + GenericArray::clone_from_slice(&buffer) + } +} + +#[cfg(test)] +mod test { + use digest::Digest; + + use super::*; + + #[test] + fn sha256_default() { + let _: Sha256 = Sha256::default(); + } + + #[test] + fn sha256_digest() { + let mut h: Sha256 = Sha256::new(); + let mut result: [u8; DIGEST256_LEN] = [0u8; DIGEST256_LEN]; + let expected = [151, 223, 53, 136, 181, 163, 242, 75, 171, 195, + 133, 27, 55, 47, 11, 167, 26, 157, 205, 222, 212, + 59, 20, 185, 208, 105, 97, 191, 193, 112, 125, 157]; + + h.input(b"foo"); + h.input(b"bar"); + h.input(b"baz"); + + result.copy_from_slice(h.fixed_result().as_slice()); + + println!("{:?}", &result[..]); + + assert_eq!(result, expected); + } + + #[test] + fn sha512_default() { + let _: Sha512 = Sha512::default(); + } + + #[test] + fn sha512_digest() { + let mut h: Sha512 = Sha512::new(); + let mut result: [u8; DIGEST512_LEN] = [0u8; DIGEST512_LEN]; + + let expected = [203, 55, 124, 16, 176, 245, 166, 44, 128, 54, 37, 167, + 153, 217, 233, 8, 190, 69, 231, 103, 245, 209, 71, 212, 116, + 73, 7, 203, 5, 89, 122, 164, 237, 211, 41, 160, 175, 20, 122, + 221, 12, 244, 24, 30, 211, 40, 250, 30, 121, 148, 38, 88, 38, + 179, 237, 61, 126, 246, 240, 103, 202, 153, 24, 90]; + + h.input(b"foo"); + h.input(b"bar"); + h.input(b"baz"); + + result.copy_from_slice(h.fixed_result().as_slice()); + + println!("{:?}", &result[..]); + + assert_eq!(&result[..], &expected[..]); + } +} diff --git a/src/rust/crypto/lib.rs b/src/rust/crypto/lib.rs new file mode 100644 index 0000000000..f72a859dd7 --- /dev/null +++ b/src/rust/crypto/lib.rs @@ -0,0 +1,45 @@ +// Copyright (c) 2018, The Tor Project, Inc. +// Copyright (c) 2018, isis agora lovecruft +// See LICENSE for licensing information + +//! Common cryptographic functions and utilities. +//! +//! # Hash Digests and eXtendable Output Functions (XOFs) +//! +//! The `digests` module contains submodules for specific hash digests +//! and extendable output functions. +//! +//! ```rust,no_run +//! use crypto::digests::sha2::*; +//! +//! let mut hasher: Sha256 = Sha256::default(); +//! let mut result: [u8; 32] = [0u8; 32]; +//! +//! hasher.input(b"foo"); +//! hasher.input(b"bar"); +//! hasher.input(b"baz"); +//! +//! result.copy_from_slice(hasher.result().as_slice()); +//! +//! assert!(result == [b'X'; DIGEST256_LEN]); +//! ``` + +#[deny(missing_docs)] + +// External crates from cargo or TOR_RUST_DEPENDENCIES. +extern crate digest; +extern crate libc; +extern crate rand_core; + +// External dependencies for tests. +#[cfg(test)] +extern crate rand as rand_crate; + +// Our local crates. +extern crate external; +#[cfg(not(test))] +#[macro_use] +extern crate tor_log; + +pub mod digests; // Unfortunately named "digests" plural to avoid name conflict with the digest crate +pub mod rand; diff --git a/src/rust/crypto/rand/mod.rs b/src/rust/crypto/rand/mod.rs new file mode 100644 index 0000000000..82d02a70bb --- /dev/null +++ b/src/rust/crypto/rand/mod.rs @@ -0,0 +1,6 @@ +// Copyright (c) 2018, The Tor Project, Inc. +// Copyright (c) 2018, isis agora lovecruft +// See LICENSE for licensing information + +// Internal dependencies +pub mod rng; diff --git a/src/rust/crypto/rand/rng.rs b/src/rust/crypto/rand/rng.rs new file mode 100644 index 0000000000..07a0a7bdc7 --- /dev/null +++ b/src/rust/crypto/rand/rng.rs @@ -0,0 +1,140 @@ +// Copyright (c) 2018, The Tor Project, Inc. +// Copyright (c) 2018, isis agora lovecruft +// See LICENSE for licensing information + +//! Wrappers for Tor's random number generators to provide implementations of +//! `rand_core` traits. + +// This is the real implementation, in use in production, which calls into our C +// wrappers in /src/common/crypto_rand.c, which call into OpenSSL, system +// libraries, and make syscalls. +#[cfg(not(test))] +mod internal { + use std::u64; + + use rand_core::CryptoRng; + use rand_core::Error; + use rand_core::RngCore; + use rand_core::impls::next_u32_via_fill; + use rand_core::impls::next_u64_via_fill; + + use external::c_tor_crypto_rand; + use external::c_tor_crypto_strongest_rand; + use external::c_tor_crypto_seed_rng; + + use tor_log::LogDomain; + use tor_log::LogSeverity; + + /// Largest strong entropy request permitted. + // + // C_RUST_COUPLED: `MAX_STRONGEST_RAND_SIZE` /src/common/crypto_rand.c + const MAX_STRONGEST_RAND_SIZE: usize = 256; + + /// A wrapper around OpenSSL's RNG. + pub struct TorRng { + // This private, zero-length field forces the struct to be treated the + // same as its opaque C couterpart. + _unused: [u8; 0], + } + + /// Mark `TorRng` as being suitable for cryptographic purposes. + impl CryptoRng for TorRng {} + + impl TorRng { + // C_RUST_COUPLED: `crypto_seed_rng()` /src/common/crypto_rand.c + #[allow(dead_code)] + pub fn new() -> Self { + if !c_tor_crypto_seed_rng() { + tor_log_msg!(LogSeverity::Warn, LogDomain::General, + "TorRng::from_seed()", + "The RNG could not be seeded!"); + } + // XXX also log success at info level —isis + TorRng{ _unused: [0u8; 0] } + } + } + + impl RngCore for TorRng { + // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c + fn next_u32(&mut self) -> u32 { + next_u32_via_fill(self) + } + + // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c + fn next_u64(&mut self) -> u64 { + next_u64_via_fill(self) + } + + // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c + fn fill_bytes(&mut self, dest: &mut [u8]) { + c_tor_crypto_rand(dest); + } + + // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c + fn try_fill_bytes(&mut self, dest: &mut [u8]) -> Result<(), Error> { + Ok(self.fill_bytes(dest)) + } + } + + /// A CSPRNG which hashes together randomness from OpenSSL's RNG and entropy + /// obtained from the operating system. + pub struct TorStrongestRng { + // This private, zero-length field forces the struct to be treated the + // same as its opaque C couterpart. + _unused: [u8; 0], + } + + /// Mark `TorRng` as being suitable for cryptographic purposes. + impl CryptoRng for TorStrongestRng {} + + impl TorStrongestRng { + // C_RUST_COUPLED: `crypto_seed_rng()` /src/common/crypto_rand.c + #[allow(dead_code)] + pub fn new() -> Self { + if !c_tor_crypto_seed_rng() { + tor_log_msg!(LogSeverity::Warn, LogDomain::General, + "TorStrongestRng::from_seed()", + "The RNG could not be seeded!"); + } + // XXX also log success at info level —isis + TorStrongestRng{ _unused: [0u8; 0] } + } + } + + impl RngCore for TorStrongestRng { + // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c + fn next_u32(&mut self) -> u32 { + next_u32_via_fill(self) + } + + // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c + fn next_u64(&mut self) -> u64 { + next_u64_via_fill(self) + } + + // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c + fn fill_bytes(&mut self, dest: &mut [u8]) { + debug_assert!(dest.len() <= MAX_STRONGEST_RAND_SIZE); + + c_tor_crypto_strongest_rand(dest); + } + + // C_RUST_COUPLED: `crypto_strongest_rand()` /src/common/crypto_rand.c + fn try_fill_bytes(&mut self, dest: &mut [u8]) -> Result<(), Error> { + Ok(self.fill_bytes(dest)) + } + } +} + +// For testing, we expose a pure-Rust implementation. +#[cfg(test)] +mod internal { + // It doesn't matter if we pretend ChaCha is a CSPRNG in tests. + pub use rand_crate::ChaChaRng as TorRng; + pub use rand_crate::ChaChaRng as TorStrongestRng; +} + +// Finally, expose the public functionality of whichever appropriate internal +// module. +pub use self::internal::*; + diff --git a/src/rust/external/Cargo.toml b/src/rust/external/Cargo.toml index b5957b1079..60ec03be40 100644 --- a/src/rust/external/Cargo.toml +++ b/src/rust/external/Cargo.toml @@ -6,6 +6,9 @@ name = "external" [dependencies] libc = "=0.2.39" +[dependencies.smartlist] +path = "../smartlist" + [lib] name = "external" path = "lib.rs" diff --git a/src/rust/external/crypto_digest.rs b/src/rust/external/crypto_digest.rs new file mode 100644 index 0000000000..3e8801f203 --- /dev/null +++ b/src/rust/external/crypto_digest.rs @@ -0,0 +1,406 @@ +// Copyright (c) 2018, The Tor Project, Inc. +// Copyright (c) 2018, isis agora lovecruft +// See LICENSE for licensing information + +//! Bindings to external digest and XOF functions which live within +//! src/common/crypto_digest.[ch]. +//! +//! We wrap our C implementations in src/common/crypto_digest.[ch] with more +//! Rusty types and interfaces in src/rust/crypto/digest/. + +use std::process::abort; + +use libc::c_char; +use libc::c_int; +use libc::size_t; +use libc::uint8_t; + +use smartlist::Stringlist; + +/// Length of the output of our message digest. +pub const DIGEST_LEN: usize = 20; + +/// Length of the output of our second (improved) message digests. (For now +/// this is just sha256, but it could be any other 256-bit digest.) +pub const DIGEST256_LEN: usize = 32; + +/// Length of the output of our 64-bit optimized message digests (SHA512). +pub const DIGEST512_LEN: usize = 64; + +/// Length of a sha1 message digest when encoded in base32 with trailing = signs +/// removed. +pub const BASE32_DIGEST_LEN: usize = 32; + +/// Length of a sha1 message digest when encoded in base64 with trailing = signs +/// removed. +pub const BASE64_DIGEST_LEN: usize = 27; + +/// Length of a sha256 message digest when encoded in base64 with trailing = +/// signs removed. +pub const BASE64_DIGEST256_LEN: usize = 43; + +/// Length of a sha512 message digest when encoded in base64 with trailing = +/// signs removed. +pub const BASE64_DIGEST512_LEN: usize = 86; + +/// Length of hex encoding of SHA1 digest, not including final NUL. +pub const HEX_DIGEST_LEN: usize = 40; + +/// Length of hex encoding of SHA256 digest, not including final NUL. +pub const HEX_DIGEST256_LEN: usize = 64; + +/// Length of hex encoding of SHA512 digest, not including final NUL. +pub const HEX_DIGEST512_LEN: usize = 128; + +/// Our C code uses an enum to declare the digest algorithm types which we know +/// about. However, because enums are implementation-defined in C, we can +/// neither work with them directly nor translate them into Rust enums. +/// Instead, we represent them as a u8 (under the assumption that we'll never +/// support more than 256 hash functions). +#[allow(non_camel_case_types)] +type digest_algorithm_t = u8; + +const DIGEST_SHA1: digest_algorithm_t = 0; +const DIGEST_SHA256: digest_algorithm_t = 1; +const DIGEST_SHA512: digest_algorithm_t = 2; +const DIGEST_SHA3_256: digest_algorithm_t = 3; +const DIGEST_SHA3_512: digest_algorithm_t = 4; + +/// The number of hash digests we produce for a `common_digests_t`. +/// +/// We can't access these from Rust, because their definitions in C require +/// introspecting the `digest_algorithm_t` typedef, which is an enum, so we have +/// to redefine them here. +const N_COMMON_DIGEST_ALGORITHMS: usize = DIGEST_SHA256 as usize + 1; + +/// A digest function. +#[repr(C)] +#[derive(Debug, Copy, Clone)] +#[allow(non_camel_case_types)] +struct crypto_digest_t { + // This private, zero-length field forces the struct to be treated the same + // as its opaque C couterpart. + _unused: [u8; 0], +} + +/// An eXtendible Output Function (XOF). +#[repr(C)] +#[derive(Debug, Copy, Clone)] +#[allow(non_camel_case_types)] +struct crypto_xof_t { + // This private, zero-length field forces the struct to be treated the same + // as its opaque C couterpart. + _unused: [u8; 0], +} + +/// A set of all the digests we commonly compute, taken on a single +/// string. Any digests that are shorter than 512 bits are right-padded +/// with 0 bits. +/// +/// Note that this representation wastes 44 bytes for the SHA1 case, so +/// don't use it for anything where we need to allocate a whole bunch at +/// once. +#[repr(C)] +#[derive(Debug, Copy, Clone)] +#[allow(non_camel_case_types)] +struct common_digests_t { + pub d: [[c_char; N_COMMON_DIGEST_ALGORITHMS]; DIGEST256_LEN], +} + +/// A `smartlist_t` is just an alias for the `#[repr(C)]` type `Stringlist`, to +/// make it more clear that we're working with a smartlist which is owned by C. +#[allow(non_camel_case_types)] +// BINDGEN_GENERATED: This type isn't actually bindgen generated, but the code +// below it which uses it is. As such, this comes up as "dead code" as well. +#[allow(dead_code)] +type smartlist_t = Stringlist; + +/// All of the external functions from `src/common/crypto_digest.h`. +/// +/// These are kept private because they should be wrapped with Rust to make their usage safer. +// +// BINDGEN_GENERATED: These definitions were generated with bindgen and cleaned +// up manually. As such, there are more bindings than are likely necessary or +// which are in use. +#[allow(dead_code)] +extern "C" { + fn crypto_digest(digest: *mut c_char, m: *const c_char, len: size_t) -> c_int; + fn crypto_digest256(digest: *mut c_char, m: *const c_char, len: size_t, + algorithm: digest_algorithm_t) -> c_int; + fn crypto_digest512(digest: *mut c_char, m: *const c_char, len: size_t, + algorithm: digest_algorithm_t) -> c_int; + fn crypto_common_digests(ds_out: *mut common_digests_t, m: *const c_char, len: size_t) -> c_int; + fn crypto_digest_smartlist_prefix(digest_out: *mut c_char, len_out: size_t, prepend: *const c_char, + lst: *const smartlist_t, append: *const c_char, alg: digest_algorithm_t); + fn crypto_digest_smartlist(digest_out: *mut c_char, len_out: size_t, + lst: *const smartlist_t, append: *const c_char, alg: digest_algorithm_t); + fn crypto_digest_algorithm_get_name(alg: digest_algorithm_t) -> *const c_char; + fn crypto_digest_algorithm_get_length(alg: digest_algorithm_t) -> size_t; + fn crypto_digest_algorithm_parse_name(name: *const c_char) -> c_int; + fn crypto_digest_new() -> *mut crypto_digest_t; + fn crypto_digest256_new(algorithm: digest_algorithm_t) -> *mut crypto_digest_t; + fn crypto_digest512_new(algorithm: digest_algorithm_t) -> *mut crypto_digest_t; + fn crypto_digest_free_(digest: *mut crypto_digest_t); + fn crypto_digest_add_bytes(digest: *mut crypto_digest_t, data: *const c_char, len: size_t); + fn crypto_digest_get_digest(digest: *mut crypto_digest_t, out: *mut c_char, out_len: size_t); + fn crypto_digest_dup(digest: *const crypto_digest_t) -> *mut crypto_digest_t; + fn crypto_digest_assign(into: *mut crypto_digest_t, from: *const crypto_digest_t); + fn crypto_hmac_sha256(hmac_out: *mut c_char, key: *const c_char, key_len: size_t, + msg: *const c_char, msg_len: size_t); + fn crypto_mac_sha3_256(mac_out: *mut uint8_t, len_out: size_t, + key: *const uint8_t, key_len: size_t, + msg: *const uint8_t, msg_len: size_t); + fn crypto_xof_new() -> *mut crypto_xof_t; + fn crypto_xof_add_bytes(xof: *mut crypto_xof_t, data: *const uint8_t, len: size_t); + fn crypto_xof_squeeze_bytes(xof: *mut crypto_xof_t, out: *mut uint8_t, len: size_t); + fn crypto_xof_free(xof: *mut crypto_xof_t); +} + +/// A wrapper around a `digest_algorithm_t`. +pub enum DigestAlgorithm { + SHA2_256, + SHA2_512, + SHA3_256, + SHA3_512, +} + +impl From<DigestAlgorithm> for digest_algorithm_t { + fn from(digest: DigestAlgorithm) -> digest_algorithm_t { + match digest { + DigestAlgorithm::SHA2_256 => DIGEST_SHA256, + DigestAlgorithm::SHA2_512 => DIGEST_SHA512, + DigestAlgorithm::SHA3_256 => DIGEST_SHA3_256, + DigestAlgorithm::SHA3_512 => DIGEST_SHA3_512, + } + } +} + +/// A wrapper around a mutable pointer to a `crypto_digest_t`. +pub struct CryptoDigest(*mut crypto_digest_t); + +/// Explicitly copy the state of a `CryptoDigest` hash digest context. +/// +/// # C_RUST_COUPLED +/// +/// * `crypto_digest_dup` +impl Clone for CryptoDigest { + fn clone(&self) -> CryptoDigest { + let digest: *mut crypto_digest_t; + + unsafe { + digest = crypto_digest_dup(self.0 as *const crypto_digest_t); + } + + // See the note in the implementation of CryptoDigest for the + // reasoning for `abort()` here. + if digest.is_null() { + abort(); + } + + CryptoDigest(digest) + } +} + +impl CryptoDigest { + /// A wrapper to call one of the C functions `crypto_digest_new`, + /// `crypto_digest256_new`, or `crypto_digest512_new`. + /// + /// # Warnings + /// + /// This function will `abort()` the entire process in an "abnormal" fashion, + /// i.e. not unwinding this or any other thread's stack, running any + /// destructors, or calling any panic/exit hooks) if `tor_malloc()` (called in + /// `crypto_digest256_new()`) is unable to allocate memory. + /// + /// # Returns + /// + /// A new `CryptoDigest`, which is a wrapper around a opaque representation + /// of a `crypto_digest_t`. The underlying `crypto_digest_t` _MUST_ only + /// ever be handled via a raw pointer, and never introspected. + /// + /// # C_RUST_COUPLED + /// + /// * `crypto_digest_new` + /// * `crypto_digest256_new` + /// * `crypto_digest512_new` + /// * `tor_malloc` (called by `crypto_digest256_new`, but we make + /// assumptions about its behvaiour and return values here) + pub fn new(algorithm: Option<DigestAlgorithm>) -> CryptoDigest { + let digest: *mut crypto_digest_t; + + if algorithm.is_none() { + unsafe { + digest = crypto_digest_new(); + } + } else { + let algo: digest_algorithm_t = algorithm.unwrap().into(); // can't fail because it's Some + + unsafe { + // XXX This is a pretty awkward API to use from Rust... + digest = match algo { + DIGEST_SHA1 => crypto_digest_new(), + DIGEST_SHA256 => crypto_digest256_new(DIGEST_SHA256), + DIGEST_SHA3_256 => crypto_digest256_new(DIGEST_SHA3_256), + DIGEST_SHA512 => crypto_digest512_new(DIGEST_SHA512), + DIGEST_SHA3_512 => crypto_digest512_new(DIGEST_SHA3_512), + _ => abort(), + } + } + } + + // In our C code, `crypto_digest*_new()` allocates memory with + // `tor_malloc()`. In `tor_malloc()`, if the underlying malloc + // implementation fails to allocate the requested memory and returns a + // NULL pointer, we call `exit(1)`. In the case that this `exit(1)` is + // called within a worker, be that a process or a thread, the inline + // comments within `tor_malloc()` mention "that's ok, since the parent + // will run out of memory soon anyway". However, if it takes long + // enough for the worker to die, and it manages to return a NULL pointer + // to our Rust code, our Rust is now in an irreparably broken state and + // may exhibit undefined behaviour. An even worse scenario, if/when we + // have parent/child processes/threads controlled by Rust, would be that + // the UB contagion in Rust manages to spread to other children before + // the entire process (hopefully terminates). + // + // However, following the assumptions made in `tor_malloc()` that + // calling `exit(1)` in a child is okay because the parent will + // eventually run into the same errors, and also to stymie any UB + // contagion in the meantime, we call abort!() here to terminate the + // entire program immediately. + if digest.is_null() { + abort(); + } + + CryptoDigest(digest) + } + + /// A wrapper to call the C function `crypto_digest_add_bytes`. + /// + /// # Inputs + /// + /// * `bytes`: a byte slice of bytes to be added into this digest. + /// + /// # C_RUST_COUPLED + /// + /// * `crypto_digest_add_bytes` + pub fn add_bytes(&self, bytes: &[u8]) { + unsafe { + crypto_digest_add_bytes(self.0 as *mut crypto_digest_t, + bytes.as_ptr() as *const c_char, + bytes.len() as size_t) + } + } +} + +impl Drop for CryptoDigest { + fn drop(&mut self) { + unsafe { + crypto_digest_free_(self.0 as *mut crypto_digest_t); + } + } +} + +/// Get the 256-bit digest output of a `crypto_digest_t`. +/// +/// # Inputs +/// +/// * `digest`: A `CryptoDigest` which wraps either a `DIGEST_SHA256` or a +/// `DIGEST_SHA3_256`. +/// +/// # Warning +/// +/// Calling this function with a `CryptoDigest` which is neither SHA2-256 or +/// SHA3-256 is a programming error. Since we cannot introspect the opaque +/// struct from Rust, however, there is no way for us to check that the correct +/// one is being passed in. That is up to you, dear programmer. If you mess +/// up, you will get a incorrectly-sized hash digest in return, and it will be +/// your fault. Don't do that. +/// +/// # Returns +/// +/// A 256-bit hash digest, as a `[u8; 32]`. +/// +/// # C_RUST_COUPLED +/// +/// * `crypto_digest_get_digest` +/// * `DIGEST256_LEN` +// +// FIXME: Once const generics land in Rust, we should genericise calling +// crypto_digest_get_digest w.r.t. output array size. +pub fn get_256_bit_digest(digest: CryptoDigest) -> [u8; DIGEST256_LEN] { + let mut buffer: [u8; DIGEST256_LEN] = [0u8; DIGEST256_LEN]; + + unsafe { + crypto_digest_get_digest(digest.0, + buffer.as_mut_ptr() as *mut c_char, + DIGEST256_LEN as size_t); + + if buffer.as_ptr().is_null() { + abort(); + } + } + buffer +} + +/// Get the 512-bit digest output of a `crypto_digest_t`. +/// +/// # Inputs +/// +/// * `digest`: A `CryptoDigest` which wraps either a `DIGEST_SHA512` or a +/// `DIGEST_SHA3_512`. +/// +/// # Warning +/// +/// Calling this function with a `CryptoDigest` which is neither SHA2-512 or +/// SHA3-512 is a programming error. Since we cannot introspect the opaque +/// struct from Rust, however, there is no way for us to check that the correct +/// one is being passed in. That is up to you, dear programmer. If you mess +/// up, you will get a incorrectly-sized hash digest in return, and it will be +/// your fault. Don't do that. +/// +/// # Returns +/// +/// A 512-bit hash digest, as a `[u8; 64]`. +/// +/// # C_RUST_COUPLED +/// +/// * `crypto_digest_get_digest` +/// * `DIGEST512_LEN` +// +// FIXME: Once const generics land in Rust, we should genericise calling +// crypto_digest_get_digest w.r.t. output array size. +pub fn get_512_bit_digest(digest: CryptoDigest) -> [u8; DIGEST512_LEN] { + let mut buffer: [u8; DIGEST512_LEN] = [0u8; DIGEST512_LEN]; + + unsafe { + crypto_digest_get_digest(digest.0, + buffer.as_mut_ptr() as *mut c_char, + DIGEST512_LEN as size_t); + + if buffer.as_ptr().is_null() { + abort(); + } + } + buffer +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_layout_common_digests_t() { + assert_eq!(::std::mem::size_of::<common_digests_t>(), 64usize, + concat!("Size of: ", stringify!(common_digests_t))); + assert_eq!(::std::mem::align_of::<common_digests_t>(), 1usize, + concat!("Alignment of ", stringify!(common_digests_t))); + } + + #[test] + fn test_layout_crypto_digest_t() { + assert_eq!(::std::mem::size_of::<crypto_digest_t>(), 0usize, + concat!("Size of: ", stringify!(crypto_digest_t))); + assert_eq!(::std::mem::align_of::<crypto_digest_t>(), 1usize, + concat!("Alignment of ", stringify!(crypto_digest_t))); + } +} diff --git a/src/rust/external/crypto_rand.rs b/src/rust/external/crypto_rand.rs new file mode 100644 index 0000000000..af1ade0161 --- /dev/null +++ b/src/rust/external/crypto_rand.rs @@ -0,0 +1,87 @@ +// Copyright (c) 2018, The Tor Project, Inc. +// Copyright (c) 2018, isis agora lovecruft +// See LICENSE for licensing information + +//! Bindings to external (P)RNG interfaces and utilities in +//! src/common/crypto_rand.[ch]. +//! +//! We wrap our C implementations in src/common/crypto_rand.[ch] here in order +//! to provide wrappers with native Rust types, and then provide more Rusty +//! types and and trait implementations in src/rust/crypto/rand/. + +use std::time::Duration; + +use libc::c_double; +use libc::c_int; +use libc::size_t; +use libc::time_t; +use libc::uint8_t; + +extern "C" { + fn crypto_seed_rng() -> c_int; + fn crypto_rand(out: *mut uint8_t, out_len: size_t); + fn crypto_strongest_rand(out: *mut uint8_t, out_len: size_t); + fn crypto_rand_time_range(min: time_t, max: time_t) -> time_t; + fn crypto_rand_double() -> c_double; +} + +/// Seed OpenSSL's random number generator with bytes from the operating +/// system. +/// +/// # Returns +/// +/// `true` on success; `false` on failure. +pub fn c_tor_crypto_seed_rng() -> bool { + let ret: c_int; + + unsafe { + ret = crypto_seed_rng(); + } + match ret { + 0 => return true, + _ => return false, + } +} + +/// Fill the bytes of `dest` with random data. +pub fn c_tor_crypto_rand(dest: &mut [u8]) { + unsafe { + crypto_rand(dest.as_mut_ptr(), dest.len() as size_t); + } +} + +/// Fill the bytes of `dest` with "strong" random data by hashing +/// together randomness obtained from OpenSSL's RNG and the operating +/// system. +pub fn c_tor_crypto_strongest_rand(dest: &mut [u8]) { + // We'll let the C side panic if the len is larger than + // MAX_STRONGEST_RAND_SIZE, rather than potentially panicking here. A + // paranoid caller should assert on the length of dest *before* calling this + // function. + unsafe { + crypto_strongest_rand(dest.as_mut_ptr(), dest.len() as size_t); + } +} + +/// Get a random time, in seconds since the Unix Epoch. +/// +/// # Returns +/// +/// A `std::time::Duration` of seconds since the Unix Epoch. +pub fn c_tor_crypto_rand_time_range(min: &Duration, max: &Duration) -> Duration { + let ret: time_t; + + unsafe { + ret = crypto_rand_time_range(min.as_secs() as time_t, max.as_secs() as time_t); + } + + Duration::from_secs(ret as u64) +} + +/// Return a pseudorandom 64-bit float, chosen uniformly from the range [0.0, 1.0). +pub fn c_tor_crypto_rand_double() -> f64 { + unsafe { + crypto_rand_double() + } +} + diff --git a/src/rust/external/lib.rs b/src/rust/external/lib.rs index 0af0d6452d..b72a4f6e4c 100644 --- a/src/rust/external/lib.rs +++ b/src/rust/external/lib.rs @@ -1,4 +1,4 @@ -//! Copyright (c) 2016-2017, The Tor Project, Inc. */ +//! Copyright (c) 2016-2018, The Tor Project, Inc. */ //! See LICENSE for licensing information */ //! Interface for external calls to tor C ABI @@ -9,6 +9,11 @@ extern crate libc; +extern crate smartlist; + +pub mod crypto_digest; +mod crypto_rand; mod external; +pub use crypto_rand::*; pub use external::*; diff --git a/src/rust/include.am b/src/rust/include.am index 7a0181e373..5e5b0b3faf 100644 --- a/src/rust/include.am +++ b/src/rust/include.am @@ -1,10 +1,19 @@ include src/rust/tor_rust/include.am EXTRA_DIST +=\ + src/rust/build.rs \ src/rust/Cargo.toml \ src/rust/Cargo.lock \ src/rust/.cargo/config.in \ + src/rust/crypto/Cargo.toml \ + src/rust/crypto/lib.rs \ + src/rust/crypto/digests/mod.rs \ + src/rust/crypto/digests/sha2.rs \ + src/rust/crypto/rand/mod.rs \ + src/rust/crypto/rand/rng.rs \ src/rust/external/Cargo.toml \ + src/rust/external/crypto_digest.rs \ + src/rust/external/crypto_rand.rs \ src/rust/external/external.rs \ src/rust/external/lib.rs \ src/rust/protover/Cargo.toml \ @@ -20,6 +29,9 @@ EXTRA_DIST +=\ src/rust/tor_allocate/Cargo.toml \ src/rust/tor_allocate/lib.rs \ src/rust/tor_allocate/tor_allocate.rs \ + src/rust/tor_log/Cargo.toml \ + src/rust/tor_log/lib.rs \ + src/rust/tor_log/tor_log.rs \ src/rust/tor_rust/Cargo.toml \ src/rust/tor_rust/include.am \ src/rust/tor_rust/lib.rs \ diff --git a/src/rust/protover/Cargo.toml b/src/rust/protover/Cargo.toml index 86301b8787..a8480e142a 100644 --- a/src/rust/protover/Cargo.toml +++ b/src/rust/protover/Cargo.toml @@ -3,6 +3,8 @@ authors = ["The Tor Project"] version = "0.0.1" name = "protover" +[features] + [dependencies] libc = "=0.2.39" @@ -18,6 +20,9 @@ path = "../tor_util" [dependencies.tor_allocate] path = "../tor_allocate" +[dependencies.tor_log] +path = "../tor_log" + [lib] name = "protover" path = "lib.rs" diff --git a/src/rust/protover/ffi.rs b/src/rust/protover/ffi.rs index f0216f157b..7386e988c5 100644 --- a/src/rust/protover/ffi.rs +++ b/src/rust/protover/ffi.rs @@ -10,9 +10,6 @@ use std::ffi::CStr; use smartlist::*; use tor_allocate::allocate_and_copy_string; -use tor_util::strings::byte_slice_is_c_like; -use tor_util::strings::empty_static_cstr; - use errors::ProtoverError; use protover::*; @@ -182,18 +179,7 @@ pub extern "C" fn protocol_list_supports_protocol_or_later( pub extern "C" fn protover_get_supported_protocols() -> *const c_char { let supported: &'static CStr; - // If we're going to pass it to C, there cannot be any intermediate NUL - // bytes. An assert is okay here, since changing the const byte slice - // in protover.rs to contain a NUL byte somewhere in the middle would be a - // programming error. - assert!(byte_slice_is_c_like(SUPPORTED_PROTOCOLS)); - - // It's okay to unwrap the result of this function because - // we can see that the bytes we're passing into it 1) are valid UTF-8, - // 2) have no intermediate NUL bytes, and 3) are terminated with a NUL - // byte. - supported = CStr::from_bytes_with_nul(SUPPORTED_PROTOCOLS).unwrap(); - + supported = get_supported_protocols_cstr(); supported.as_ptr() } @@ -252,10 +238,9 @@ pub extern "C" fn protover_is_supported_here( #[no_mangle] pub extern "C" fn protover_compute_for_old_tor(version: *const c_char) -> *const c_char { let supported: &'static CStr; - let elder_protocols: &'static [u8]; let empty: &'static CStr; - empty = empty_static_cstr(); + empty = cstr!(""); if version.is_null() { return empty.as_ptr(); @@ -270,19 +255,6 @@ pub extern "C" fn protover_compute_for_old_tor(version: *const c_char) -> *const Err(_) => return empty.as_ptr(), }; - elder_protocols = compute_for_old_tor_cstr(&version); - - // If we're going to pass it to C, there cannot be any intermediate NUL - // bytes. An assert is okay here, since changing the const byte slice - // in protover.rs to contain a NUL byte somewhere in the middle would be a - // programming error. - assert!(byte_slice_is_c_like(elder_protocols)); - - // It's okay to unwrap the result of this function because - // we can see that the bytes we're passing into it 1) are valid UTF-8, - // 2) have no intermediate NUL bytes, and 3) are terminated with a NUL - // byte. - supported = CStr::from_bytes_with_nul(elder_protocols).unwrap_or(empty); - + supported = compute_for_old_tor_cstr(&version); supported.as_ptr() } diff --git a/src/rust/protover/lib.rs b/src/rust/protover/lib.rs index 483260bca8..ce964196fd 100644 --- a/src/rust/protover/lib.rs +++ b/src/rust/protover/lib.rs @@ -28,6 +28,7 @@ extern crate libc; extern crate smartlist; extern crate external; extern crate tor_allocate; +#[macro_use] extern crate tor_util; pub mod errors; diff --git a/src/rust/protover/protover.rs b/src/rust/protover/protover.rs index c11c7c1803..68027056c4 100644 --- a/src/rust/protover/protover.rs +++ b/src/rust/protover/protover.rs @@ -3,12 +3,12 @@ use std::collections::HashMap; use std::collections::hash_map; +use std::ffi::CStr; use std::fmt; use std::str; use std::str::FromStr; use std::string::String; -use tor_util::strings::NUL_BYTE; use external::c_tor_version_as_new_as; use errors::ProtoverError; @@ -31,30 +31,6 @@ const MAX_PROTOCOLS_TO_EXPAND: usize = (1<<16); /// The maximum size an `UnknownProtocol`'s name may be. pub(crate) const MAX_PROTOCOL_NAME_LENGTH: usize = 100; -/// Currently supported protocols and their versions, as a byte-slice. -/// -/// # Warning -/// -/// This byte-slice ends in a NUL byte. This is so that we can directly convert -/// it to an `&'static CStr` in the FFI code, in order to hand the static string -/// to C in a way that is compatible with C static strings. -/// -/// Rust code which wishes to accesses this string should use -/// `protover::get_supported_protocols()` instead. -/// -/// C_RUST_COUPLED: src/or/protover.c `protover_get_supported_protocols` -pub(crate) const SUPPORTED_PROTOCOLS: &'static [u8] = - b"Cons=1-2 \ - Desc=1-2 \ - DirCache=1-2 \ - HSDir=1-2 \ - HSIntro=3-4 \ - HSRend=1-2 \ - Link=1-5 \ - LinkAuth=1,3 \ - Microdesc=1-2 \ - Relay=1-2\0"; - /// Known subprotocols in Tor. Indicates which subprotocol a relay supports. /// /// C_RUST_COUPLED: src/or/protover.h `protocol_type_t` @@ -148,21 +124,33 @@ impl From<Protocol> for UnknownProtocol { } } -/// Get the string representation of current supported protocols +/// Get a CStr representation of current supported protocols, for +/// passing to C, or for converting to a `&str` for Rust. /// /// # Returns /// -/// A `String` whose value is the existing protocols supported by tor. +/// An `&'static CStr` whose value is the existing protocols supported by tor. /// Returned data is in the format as follows: /// /// "HSDir=1-1 LinkAuth=1" /// -pub fn get_supported_protocols() -> &'static str { - // The `len() - 1` is to remove the NUL byte. - // The `unwrap` is safe becauase we SUPPORTED_PROTOCOLS is under - // our control. - str::from_utf8(&SUPPORTED_PROTOCOLS[..SUPPORTED_PROTOCOLS.len() - 1]) - .unwrap_or("") +/// # Note +/// +/// Rust code can use the `&'static CStr` as a normal `&'a str` by +/// calling `protover::get_supported_protocols`. +/// +// C_RUST_COUPLED: src/or/protover.c `protover_get_supported_protocols` +pub(crate) fn get_supported_protocols_cstr() -> &'static CStr { + cstr!("Cons=1-2 \ + Desc=1-2 \ + DirCache=1-2 \ + HSDir=1-2 \ + HSIntro=3-4 \ + HSRend=1-2 \ + Link=1-5 \ + LinkAuth=1,3 \ + Microdesc=1-2 \ + Relay=1-2") } /// A map of protocol names to the versions of them which are supported. @@ -185,7 +173,8 @@ impl ProtoEntry { /// ProtoEntry, which is useful when looking up a specific /// subprotocol. pub fn supported() -> Result<Self, ProtoverError> { - let supported: &'static str = get_supported_protocols(); + let supported_cstr: &'static CStr = get_supported_protocols_cstr(); + let supported: &str = supported_cstr.to_str().unwrap_or(""); supported.parse() } @@ -703,7 +692,7 @@ pub fn is_supported_here(proto: &Protocol, vers: &Version) -> bool { /// /// # Returns /// -/// A `&'static [u8]` encoding a list of protocol names and supported +/// A `&'static CStr` encoding a list of protocol names and supported /// versions. The string takes the following format: /// /// "HSDir=1-1 LinkAuth=1" @@ -719,24 +708,25 @@ pub fn is_supported_here(proto: &Protocol, vers: &Version) -> bool { /// like to use this code in Rust, please see `compute_for_old_tor()`. // // C_RUST_COUPLED: src/rust/protover.c `compute_for_old_tor` -pub(crate) fn compute_for_old_tor_cstr(version: &str) -> &'static [u8] { +pub(crate) fn compute_for_old_tor_cstr(version: &str) -> &'static CStr { + let empty: &'static CStr = cstr!(""); + if c_tor_version_as_new_as(version, FIRST_TOR_VERSION_TO_ADVERTISE_PROTOCOLS) { - return NUL_BYTE; + return empty; } if c_tor_version_as_new_as(version, "0.2.9.1-alpha") { - return b"Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1-2 \ - Link=1-4 LinkAuth=1 Microdesc=1-2 Relay=1-2\0"; + return cstr!("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1-2 \ + Link=1-4 LinkAuth=1 Microdesc=1-2 Relay=1-2"); } if c_tor_version_as_new_as(version, "0.2.7.5") { - return b"Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 \ - Link=1-4 LinkAuth=1 Microdesc=1-2 Relay=1-2\0"; + return cstr!("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 \ + Link=1-4 LinkAuth=1 Microdesc=1-2 Relay=1-2"); } if c_tor_version_as_new_as(version, "0.2.4.19") { - return b"Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 \ - Link=1-4 LinkAuth=1 Microdesc=1 Relay=1-2\0"; + return cstr!("Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 \ + Link=1-4 LinkAuth=1 Microdesc=1 Relay=1-2"); } - - NUL_BYTE + empty } /// Since older versions of Tor cannot infer their own subprotocols, @@ -767,14 +757,9 @@ pub(crate) fn compute_for_old_tor_cstr(version: &str) -> &'static [u8] { // // C_RUST_COUPLED: src/rust/protover.c `compute_for_old_tor` pub fn compute_for_old_tor(version: &str) -> Result<&'static str, ProtoverError> { - let mut computed: &'static [u8] = compute_for_old_tor_cstr(version); - - // Remove the NULL byte at the end. - computed = &computed[..computed.len() - 1]; - - // .from_utf8() fails with a Utf8Error if it couldn't validate the + // .to_str() fails with a Utf8Error if it couldn't validate the // utf-8, so convert that here into an Unparseable ProtoverError. - str::from_utf8(computed).or(Err(ProtoverError::Unparseable)) + compute_for_old_tor_cstr(version).to_str().or(Err(ProtoverError::Unparseable)) } #[cfg(test)] diff --git a/src/rust/tor_allocate/tor_allocate.rs b/src/rust/tor_allocate/tor_allocate.rs index 359df1cd7a..3c0037f139 100644 --- a/src/rust/tor_allocate/tor_allocate.rs +++ b/src/rust/tor_allocate/tor_allocate.rs @@ -1,12 +1,17 @@ // Copyright (c) 2016-2017, The Tor Project, Inc. */ // See LICENSE for licensing information */ +// No-op defined purely for testing at the module level +use libc::c_char; -use libc::{c_char, c_void}; +#[cfg(not(feature = "testing"))] use std::{ptr, slice, mem}; +use libc::c_void; -#[cfg(not(test))] -extern "C" { - fn tor_malloc_(size: usize) -> *mut c_void; +// Define a no-op implementation for testing Rust modules without linking to C +#[cfg(feature = "testing")] +pub fn allocate_and_copy_string(s: &String) -> *mut c_char { + use std::ffi::CString; + CString::new(s.as_str()).unwrap().into_raw() } // Defined only for tests, used for testing purposes, so that we don't need @@ -17,6 +22,11 @@ unsafe extern "C" fn tor_malloc_(size: usize) -> *mut c_void { malloc(size) } +#[cfg(all(not(test), not(feature = "testing")))] +extern "C" { + fn tor_malloc_(size: usize) -> *mut c_void; +} + /// Allocate memory using tor_malloc_ and copy an existing string into the /// allocated buffer, returning a pointer that can later be called in C. /// @@ -28,6 +38,7 @@ unsafe extern "C" fn tor_malloc_(size: usize) -> *mut c_void { /// /// A `*mut c_char` that should be freed by tor_free in C /// +#[cfg(not(feature = "testing"))] pub fn allocate_and_copy_string(src: &String) -> *mut c_char { let bytes: &[u8] = src.as_bytes(); diff --git a/src/rust/tor_log/Cargo.toml b/src/rust/tor_log/Cargo.toml new file mode 100644 index 0000000000..9d06299c05 --- /dev/null +++ b/src/rust/tor_log/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "tor_log" +version = "0.1.0" +authors = ["The Tor Project"] + +[lib] +name = "tor_log" +path = "lib.rs" +crate_type = ["rlib", "staticlib"] + +[features] + +[dependencies] +libc = "0.2.39" + +[dependencies.tor_allocate] +path = "../tor_allocate" diff --git a/src/rust/tor_log/lib.rs b/src/rust/tor_log/lib.rs new file mode 100644 index 0000000000..72f9e38339 --- /dev/null +++ b/src/rust/tor_log/lib.rs @@ -0,0 +1,16 @@ +//! Copyright (c) 2016-2017, The Tor Project, Inc. */ +//! See LICENSE for licensing information */ + +//! Logging wrapper for Rust to utilize Tor's logger, found at +//! src/common/log.c and src/common/torlog.h +//! +//! Exposes different interfaces depending on whether we are running in test +//! or non-test mode. When testing, we use a no-op implementation, +//! otherwise we link directly to C. + +extern crate libc; +extern crate tor_allocate; + +mod tor_log; + +pub use tor_log::*; diff --git a/src/rust/tor_log/tor_log.rs b/src/rust/tor_log/tor_log.rs new file mode 100644 index 0000000000..ad6725f0f2 --- /dev/null +++ b/src/rust/tor_log/tor_log.rs @@ -0,0 +1,270 @@ +// Copyright (c) 2016-2017, The Tor Project, Inc. */ +// See LICENSE for licensing information */ + +// Note that these functions are untested due to the fact that there are no +// return variables to test and they are calling into a C API. + +/// The related domain which the logging message is relevant. For example, +/// log messages relevant to networking would use LogDomain::LdNet, whereas +/// general messages can use LdGeneral. +#[derive(Eq, PartialEq)] +pub enum LogDomain { + Net, + General, +} + +/// The severity level at which to log messages. +#[derive(Eq, PartialEq)] +pub enum LogSeverity { + Notice, + Warn, +} + +/// Main entry point for Rust modules to log messages. +/// +/// # Inputs +/// +/// * A `severity` of type LogSeverity, which defines the level of severity the +/// message will be logged. +/// * A `domain` of type LogDomain, which defines the domain the log message +/// will be associated with. +/// * A `function` of type &str, which defines the name of the function where +/// the message is being logged. There is a current RFC for a macro that +/// defines function names. When it is, we should use it. See +/// https://github.com/rust-lang/rfcs/pull/1719 +/// * A `message` of type &str, which is the log message itself. +#[macro_export] +macro_rules! tor_log_msg { + ($severity: path, + $domain: path, + $function: expr, + $($message:tt)*) => + { + { + let msg = format!($($message)*); + $crate::tor_log_msg_impl($severity, $domain, $function, msg) + } + }; +} + +#[inline] +pub fn tor_log_msg_impl( + severity: LogSeverity, + domain: LogDomain, + function: &str, + message: String, +) { + use std::ffi::CString; + + /// Default function name to log in case of errors when converting + /// a function name to a CString + const ERR_LOG_FUNCTION: &str = "tor_log_msg"; + + /// Default message to log in case of errors when converting a log + /// message to a CString + const ERR_LOG_MSG: &str = "Unable to log message from Rust \ + module due to error when converting to CString"; + + let func = match CString::new(function) { + Ok(n) => n, + Err(_) => CString::new(ERR_LOG_FUNCTION).unwrap(), + }; + + let msg = match CString::new(message) { + Ok(n) => n, + Err(_) => CString::new(ERR_LOG_MSG).unwrap(), + }; + + // Bind to a local variable to preserve ownership. This is essential so + // that ownership is guaranteed until these local variables go out of scope + let func_ptr = func.as_ptr(); + let msg_ptr = msg.as_ptr(); + + let c_severity = unsafe { log::translate_severity(severity) }; + let c_domain = unsafe { log::translate_domain(domain) }; + + unsafe { log::tor_log_string(c_severity, c_domain, func_ptr, msg_ptr) } +} + +/// This implementation is used when compiling for actual use, as opposed to +/// testing. +#[cfg(not(test))] +pub mod log { + use libc::{c_char, c_int}; + use super::LogDomain; + use super::LogSeverity; + + /// Severity log types. These mirror definitions in /src/common/torlog.h + /// C_RUST_COUPLED: src/common/log.c, log domain types + extern "C" { + static LOG_WARN_: c_int; + static LOG_NOTICE_: c_int; + } + + /// Domain log types. These mirror definitions in /src/common/torlog.h + /// C_RUST_COUPLED: src/common/log.c, log severity types + extern "C" { + static LD_NET_: u32; + static LD_GENERAL_: u32; + } + + /// Translate Rust defintions of log domain levels to C. This exposes a 1:1 + /// mapping between types. + #[inline] + pub unsafe fn translate_domain(domain: LogDomain) -> u32 { + match domain { + LogDomain::Net => LD_NET_, + LogDomain::General => LD_GENERAL_, + } + } + + /// Translate Rust defintions of log severity levels to C. This exposes a + /// 1:1 mapping between types. + #[inline] + pub unsafe fn translate_severity(severity: LogSeverity) -> c_int { + match severity { + LogSeverity::Warn => LOG_WARN_, + LogSeverity::Notice => LOG_NOTICE_, + } + } + + /// The main entry point into Tor's logger. When in non-test mode, this + /// will link directly with `tor_log_string` in /src/or/log.c + extern "C" { + pub fn tor_log_string( + severity: c_int, + domain: u32, + function: *const c_char, + string: *const c_char, + ); + } +} + +/// This module exposes no-op functionality for testing other Rust modules +/// without linking to C. +#[cfg(test)] +pub mod log { + use libc::{c_char, c_int}; + use super::LogDomain; + use super::LogSeverity; + + pub static mut LAST_LOGGED_FUNCTION: *mut String = 0 as *mut String; + pub static mut LAST_LOGGED_MESSAGE: *mut String = 0 as *mut String; + + pub unsafe fn tor_log_string( + _severity: c_int, + _domain: u32, + function: *const c_char, + message: *const c_char, + ) { + use std::ffi::CStr; + + let f = CStr::from_ptr(function); + let fct = match f.to_str() { + Ok(n) => n, + Err(_) => "", + }; + LAST_LOGGED_FUNCTION = Box::into_raw(Box::new(String::from(fct))); + + let m = CStr::from_ptr(message); + let msg = match m.to_str() { + Ok(n) => n, + Err(_) => "", + }; + LAST_LOGGED_MESSAGE = Box::into_raw(Box::new(String::from(msg))); + } + + pub unsafe fn translate_domain(_domain: LogDomain) -> u32 { + 1 + } + + pub unsafe fn translate_severity(_severity: LogSeverity) -> c_int { + 1 + } +} + +#[cfg(test)] +mod test { + use tor_log::*; + use tor_log::log::{LAST_LOGGED_FUNCTION, LAST_LOGGED_MESSAGE}; + + #[test] + fn test_get_log_message() { + { + fn test_macro() { + tor_log_msg!( + LogSeverity::Warn, + LogDomain::Net, + "test_macro", + "test log message {}", + "a", + ); + } + + test_macro(); + + let function = unsafe { Box::from_raw(LAST_LOGGED_FUNCTION) }; + assert_eq!("test_macro", *function); + + let message = unsafe { Box::from_raw(LAST_LOGGED_MESSAGE) }; + assert_eq!("test log message a", *message); + } + + // test multiple inputs into the log message + { + fn test_macro() { + tor_log_msg!( + LogSeverity::Warn, + LogDomain::Net, + "next_test_macro", + "test log message {} {} {} {} {}", + 1, + 2, + 3, + 4, + 5 + ); + } + + test_macro(); + + let function = unsafe { Box::from_raw(LAST_LOGGED_FUNCTION) }; + assert_eq!("next_test_macro", *function); + + let message = unsafe { Box::from_raw(LAST_LOGGED_MESSAGE) }; + assert_eq!("test log message 1 2 3 4 5", *message); + } + + // test how a long log message will be formatted + { + fn test_macro() { + tor_log_msg!( + LogSeverity::Warn, + LogDomain::Net, + "test_macro", + "{}", + "All the world's a stage, and all the men and women \ + merely players: they have their exits and their \ + entrances; and one man in his time plays many parts, his \ + acts being seven ages." + ); + } + + test_macro(); + + let expected_string = "All the world's a \ + stage, and all the men \ + and women merely players: \ + they have their exits and \ + their entrances; and one man \ + in his time plays many parts, \ + his acts being seven ages."; + + let function = unsafe { Box::from_raw(LAST_LOGGED_FUNCTION) }; + assert_eq!("test_macro", *function); + + let message = unsafe { Box::from_raw(LAST_LOGGED_MESSAGE) }; + assert_eq!(expected_string, *message); + } + } +} diff --git a/src/rust/tor_rust/include.am b/src/rust/tor_rust/include.am index 40511bf9f2..99f3ede653 100644 --- a/src/rust/tor_rust/include.am +++ b/src/rust/tor_rust/include.am @@ -4,7 +4,7 @@ EXTRA_DIST +=\ EXTRA_CARGO_OPTIONS= -src/rust/target/release/@TOR_RUST_STATIC_NAME@: FORCE +@TOR_RUST_LIB_PATH@: FORCE ( cd "$(abs_top_builddir)/src/rust" ; \ CARGO_TARGET_DIR="$(abs_top_builddir)/src/rust/target" \ $(CARGO) build --release $(EXTRA_CARGO_OPTIONS) \ @@ -20,7 +20,7 @@ distclean-rust: rm -rf "$(abs_top_builddir)/src/rust/registry" if USE_RUST -build-rust: src/rust/target/release/@TOR_RUST_STATIC_NAME@ +build-rust: @TOR_RUST_LIB_PATH@ else build-rust: endif diff --git a/src/rust/tor_util/Cargo.toml b/src/rust/tor_util/Cargo.toml index b540d8c847..a606a280b2 100644 --- a/src/rust/tor_util/Cargo.toml +++ b/src/rust/tor_util/Cargo.toml @@ -11,6 +11,9 @@ crate_type = ["rlib", "staticlib"] [dependencies.tor_allocate] path = "../tor_allocate" +[dependencies.tor_log] +path = "../tor_log" + [dependencies] libc = "=0.2.39" diff --git a/src/rust/tor_util/ffi.rs b/src/rust/tor_util/ffi.rs index 5c3cdba4be..32779ed476 100644 --- a/src/rust/tor_util/ffi.rs +++ b/src/rust/tor_util/ffi.rs @@ -5,8 +5,7 @@ //! called from C. //! -use libc::c_char; -use tor_allocate::allocate_and_copy_string; +use tor_log::{LogSeverity, LogDomain}; /// Returns a short string to announce Rust support during startup. /// @@ -17,10 +16,12 @@ use tor_allocate::allocate_and_copy_string; /// tor_free(rust_str); /// ``` #[no_mangle] -pub extern "C" fn rust_welcome_string() -> *mut c_char { - let rust_welcome = String::from( +pub extern "C" fn rust_log_welcome_string() { + tor_log_msg!( + LogSeverity::Notice, + LogDomain::General, + "rust_log_welcome_string", "Tor is running with Rust integration. Please report \ - any bugs you encounter.", + any bugs you encounter." ); - allocate_and_copy_string(&rust_welcome) } diff --git a/src/rust/tor_util/lib.rs b/src/rust/tor_util/lib.rs index 12cb3896b6..94697b6069 100644 --- a/src/rust/tor_util/lib.rs +++ b/src/rust/tor_util/lib.rs @@ -7,5 +7,8 @@ extern crate libc; extern crate tor_allocate; +#[macro_use] +extern crate tor_log; + pub mod ffi; pub mod strings; diff --git a/src/rust/tor_util/strings.rs b/src/rust/tor_util/strings.rs index 9321ce4f85..505191d913 100644 --- a/src/rust/tor_util/strings.rs +++ b/src/rust/tor_util/strings.rs @@ -3,80 +3,138 @@ //! Utilities for working with static strings. -use std::ffi::CStr; - -/// A byte-array containing a single NUL byte (`b"\0"`). -pub const NUL_BYTE: &'static [u8] = b"\0"; - -/// Determine if a byte slice is a C-like string. -/// -/// These checks guarantee that: -/// -/// 1. there are no intermediate NUL bytes -/// 2. the last byte *is* a NUL byte +/// Create a `CStr` from a literal byte slice, appending a NUL byte to it first. /// /// # Warning /// -/// This function does _not_ guarantee that the bytes represent any valid -/// encoding such as ASCII or UTF-8. +/// The literal byte slice which is taken as an argument *MUST NOT* have any NUL +/// bytes (`b"\0"`) in it, anywhere, or else an empty string will be returned +/// (`CStr::from_bytes_with_nul_unchecked(b"\0")`) so as to avoid `panic!()`ing. /// /// # Examples /// /// ``` -/// # use tor_util::strings::byte_slice_is_c_like; -/// # -/// let bytes: &[u8] = b"foo bar baz"; +/// #[macro_use] +/// extern crate tor_util; /// -/// assert!(byte_slice_is_c_like(&bytes) == false); +/// use std::ffi::CStr; /// -/// let bytes: &[u8] = b"foo\0bar baz"; +/// # fn do_test() -> Result<&'static CStr, &'static str> { +/// let message: &'static str = "This is a test of the tsunami warning system."; +/// let tuesday: &'static CStr; +/// let original: &str; /// -/// assert!(byte_slice_is_c_like(&bytes) == false); +/// tuesday = cstr!("This is a test of the tsunami warning system."); +/// original = tuesday.to_str().or(Err("Couldn't unwrap CStr!"))?; /// -/// let bytes: &[u8] = b"foo bar baz\0"; +/// assert!(original == message); +/// # +/// # Ok(tuesday) +/// # } +/// # fn main() { +/// # do_test(); // so that we can use the ? operator in the test +/// # } +/// ``` +/// It is also possible to pass several string literals to this macro. They +/// will be concatenated together in the order of the arguments, unmodified, +/// before finally being suffixed with a NUL byte: /// -/// assert!(byte_slice_is_c_like(&bytes) == true); /// ``` -pub fn byte_slice_is_c_like(bytes: &[u8]) -> bool { - if !bytes[..bytes.len() - 1].contains(&0x00) && bytes[bytes.len() - 1] == 0x00 { - return true; - } - false -} - -/// Get a static `CStr` containing a single `NUL_BYTE`. +/// #[macro_use] +/// extern crate tor_util; +/// # +/// # use std::ffi::CStr; +/// # +/// # fn do_test() -> Result<&'static CStr, &'static str> { /// -/// # Examples +/// let quux: &'static CStr = cstr!("foo", "bar", "baz"); +/// let orig: &'static str = quux.to_str().or(Err("Couldn't unwrap CStr!"))?; /// -/// When used as follows in a Rust FFI function, which could be called -/// from C: +/// assert!(orig == "foobarbaz"); +/// # Ok(quux) +/// # } +/// # fn main() { +/// # do_test(); // so that we can use the ? operator in the test +/// # } +/// ``` +/// This is useful for passing static strings to C from Rust FFI code. To do so +/// so, use the `.as_ptr()` method on the resulting `&'static CStr` to convert +/// it to the Rust equivalent of a C `const char*`: /// /// ``` -/// # extern crate libc; -/// # extern crate tor_util; -/// # -/// # use tor_util::strings::empty_static_cstr; -/// use libc::c_char; +/// #[macro_use] +/// extern crate tor_util; +/// /// use std::ffi::CStr; +/// use std::os::raw::c_char; /// -/// pub extern "C" fn give_c_code_an_empty_static_string() -> *const c_char { -/// let empty: &'static CStr = empty_static_cstr(); +/// pub extern "C" fn give_static_borrowed_string_to_c() -> *const c_char { +/// let hello: &'static CStr = cstr!("Hello, language my parents wrote."); /// -/// empty.as_ptr() +/// hello.as_ptr() /// } -/// /// # fn main() { -/// # give_c_code_an_empty_static_string(); +/// # let greetings = give_static_borrowed_string_to_c(); /// # } /// ``` +/// Note that the C code this static borrowed string is passed to *MUST NOT* +/// attempt to free the memory for the string. +/// +/// # Note +/// +/// An unfortunate limitation of the rustc compiler (as of 1.25.0-nightly), is +/// that the first example above compiles, but if we were to change the +/// assignment of `tuesday` as follows, it will fail to compile, because Rust +/// macros are expanded at parse time, and at parse time there is no symbol +/// table available. /// -/// This equates to an "empty" `const char*` static string in C. -pub fn empty_static_cstr() -> &'static CStr { - let empty: &'static CStr; +/// ```ignore +/// tuesday = cstr!(message); +/// ``` +/// with the error message `error: expected a literal`. +/// +/// # Returns +/// +/// If the string literals passed as arguments contain no NUL bytes anywhere, +/// then an `&'static CStr` containing the (concatenated) bytes of the string +/// literal(s) passed as arguments, with a NUL byte appended, is returned. +/// Otherwise, an `&'static CStr` containing a single NUL byte is returned (an +/// "empty" string in C). +#[macro_export] +macro_rules! cstr { + ($($bytes:expr),*) => ( + ::std::ffi::CStr::from_bytes_with_nul( + concat!($($bytes),*, "\0").as_bytes() + ).unwrap_or( + unsafe{ + ::std::ffi::CStr::from_bytes_with_nul_unchecked(b"\0") + } + ) + ) +} + +#[cfg(test)] +mod test { + use std::ffi::CStr; + + #[test] + fn cstr_macro() { + let _: &'static CStr = cstr!("boo"); + } + + #[test] + fn cstr_macro_multi_input() { + let quux: &'static CStr = cstr!("foo", "bar", "baz"); - unsafe { - empty = CStr::from_bytes_with_nul_unchecked(NUL_BYTE); + assert!(quux.to_str().unwrap() == "foobarbaz"); } - empty + #[test] + fn cstr_macro_bad_input() { + let waving: &'static CStr = cstr!("waving not drowning o/"); + let drowning: &'static CStr = cstr!("\0 drowning not waving"); + + assert!(waving.to_str().unwrap() == "waving not drowning o/"); + assert!(drowning.to_str().unwrap() == "") + } } diff --git a/src/test/bench.c b/src/test/bench.c index 92d7a244f7..9ab23c9921 100644 --- a/src/test/bench.c +++ b/src/test/bench.c @@ -12,7 +12,7 @@ #include "or.h" #include "onion_tap.h" -#include "relay.h" +#include "relay_crypto.h" #include <openssl/opensslv.h> #include <openssl/evp.h> #include <openssl/ec.h> @@ -23,6 +23,7 @@ #include "crypto_curve25519.h" #include "onion_ntor.h" #include "crypto_ed25519.h" +#include "crypto_rand.h" #include "consdiff.h" #if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID) @@ -505,10 +506,10 @@ bench_cell_ops(void) char key1[CIPHER_KEY_LEN], key2[CIPHER_KEY_LEN]; crypto_rand(key1, sizeof(key1)); crypto_rand(key2, sizeof(key2)); - or_circ->p_crypto = crypto_cipher_new(key1); - or_circ->n_crypto = crypto_cipher_new(key2); - or_circ->p_digest = crypto_digest_new(); - or_circ->n_digest = crypto_digest_new(); + or_circ->crypto.f_crypto = crypto_cipher_new(key1); + or_circ->crypto.b_crypto = crypto_cipher_new(key2); + or_circ->crypto.f_digest = crypto_digest_new(); + or_circ->crypto.b_digest = crypto_digest_new(); reset_perftime(); @@ -518,7 +519,8 @@ bench_cell_ops(void) for (i = 0; i < iters; ++i) { char recognized = 0; crypt_path_t *layer_hint = NULL; - relay_crypt(TO_CIRCUIT(or_circ), cell, d, &layer_hint, &recognized); + relay_decrypt_cell(TO_CIRCUIT(or_circ), cell, d, + &layer_hint, &recognized); } end = perftime(); printf("%sbound cells: %.2f ns per cell. (%.2f ns per byte of payload)\n", @@ -527,10 +529,7 @@ bench_cell_ops(void) NANOCOUNT(start,end,iters*CELL_PAYLOAD_SIZE)); } - crypto_digest_free(or_circ->p_digest); - crypto_digest_free(or_circ->n_digest); - crypto_cipher_free(or_circ->p_crypto); - crypto_cipher_free(or_circ->n_crypto); + relay_crypto_clear(&or_circ->crypto); tor_free(or_circ); tor_free(cell); } diff --git a/src/test/fuzz/include.am b/src/test/fuzz/include.am index cd16dc05be..39d6d3c17b 100644 --- a/src/test/fuzz/include.am +++ b/src/test/fuzz/include.am @@ -17,8 +17,9 @@ FUZZING_LIBS = \ src/trunnel/libor-trunnel-testing.a \ $(rust_ldadd) \ @TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \ - @TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \ - @TOR_LIB_WS32@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ @CURVE25519_LIBS@ \ + @TOR_LIBEVENT_LIBS@ \ + @TOR_OPENSSL_LIBS@ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \ + @CURVE25519_LIBS@ \ @TOR_SYSTEMD_LIBS@ \ @TOR_LZMA_LIBS@ \ @TOR_ZSTD_LIBS@ @@ -46,6 +47,7 @@ LIBOSS_FUZZ_CPPFLAGS = $(FUZZING_CPPFLAGS) -DLLVM_FUZZ LIBOSS_FUZZ_CFLAGS = $(FUZZING_CFLAGS) # ===== AFL fuzzers +if UNITTESTS_ENABLED src_test_fuzz_fuzz_consensus_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_consensus.c @@ -53,7 +55,9 @@ src_test_fuzz_fuzz_consensus_CPPFLAGS = $(FUZZING_CPPFLAGS) src_test_fuzz_fuzz_consensus_CFLAGS = $(FUZZING_CFLAGS) src_test_fuzz_fuzz_consensus_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_consensus_LDADD = $(FUZZING_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_fuzz_descriptor_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_descriptor.c @@ -61,7 +65,9 @@ src_test_fuzz_fuzz_descriptor_CPPFLAGS = $(FUZZING_CPPFLAGS) src_test_fuzz_fuzz_descriptor_CFLAGS = $(FUZZING_CFLAGS) src_test_fuzz_fuzz_descriptor_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_descriptor_LDADD = $(FUZZING_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_fuzz_diff_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_diff.c @@ -69,7 +75,9 @@ src_test_fuzz_fuzz_diff_CPPFLAGS = $(FUZZING_CPPFLAGS) src_test_fuzz_fuzz_diff_CFLAGS = $(FUZZING_CFLAGS) src_test_fuzz_fuzz_diff_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_diff_LDADD = $(FUZZING_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_fuzz_diff_apply_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_diff_apply.c @@ -77,7 +85,9 @@ src_test_fuzz_fuzz_diff_apply_CPPFLAGS = $(FUZZING_CPPFLAGS) src_test_fuzz_fuzz_diff_apply_CFLAGS = $(FUZZING_CFLAGS) src_test_fuzz_fuzz_diff_apply_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_diff_apply_LDADD = $(FUZZING_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_fuzz_extrainfo_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_extrainfo.c @@ -85,7 +95,9 @@ src_test_fuzz_fuzz_extrainfo_CPPFLAGS = $(FUZZING_CPPFLAGS) src_test_fuzz_fuzz_extrainfo_CFLAGS = $(FUZZING_CFLAGS) src_test_fuzz_fuzz_extrainfo_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_extrainfo_LDADD = $(FUZZING_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_fuzz_hsdescv2_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_hsdescv2.c @@ -93,7 +105,9 @@ src_test_fuzz_fuzz_hsdescv2_CPPFLAGS = $(FUZZING_CPPFLAGS) src_test_fuzz_fuzz_hsdescv2_CFLAGS = $(FUZZING_CFLAGS) src_test_fuzz_fuzz_hsdescv2_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_hsdescv2_LDADD = $(FUZZING_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_fuzz_hsdescv3_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_hsdescv3.c @@ -101,7 +115,9 @@ src_test_fuzz_fuzz_hsdescv3_CPPFLAGS = $(FUZZING_CPPFLAGS) src_test_fuzz_fuzz_hsdescv3_CFLAGS = $(FUZZING_CFLAGS) src_test_fuzz_fuzz_hsdescv3_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_hsdescv3_LDADD = $(FUZZING_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_fuzz_http_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_http.c @@ -109,7 +125,9 @@ src_test_fuzz_fuzz_http_CPPFLAGS = $(FUZZING_CPPFLAGS) src_test_fuzz_fuzz_http_CFLAGS = $(FUZZING_CFLAGS) src_test_fuzz_fuzz_http_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_http_LDADD = $(FUZZING_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_fuzz_http_connect_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_http_connect.c @@ -117,7 +135,9 @@ src_test_fuzz_fuzz_http_connect_CPPFLAGS = $(FUZZING_CPPFLAGS) src_test_fuzz_fuzz_http_connect_CFLAGS = $(FUZZING_CFLAGS) src_test_fuzz_fuzz_http_connect_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_http_connect_LDADD = $(FUZZING_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_fuzz_iptsv2_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_iptsv2.c @@ -125,7 +145,9 @@ src_test_fuzz_fuzz_iptsv2_CPPFLAGS = $(FUZZING_CPPFLAGS) src_test_fuzz_fuzz_iptsv2_CFLAGS = $(FUZZING_CFLAGS) src_test_fuzz_fuzz_iptsv2_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_iptsv2_LDADD = $(FUZZING_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_fuzz_microdesc_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_microdesc.c @@ -133,7 +155,9 @@ src_test_fuzz_fuzz_microdesc_CPPFLAGS = $(FUZZING_CPPFLAGS) src_test_fuzz_fuzz_microdesc_CFLAGS = $(FUZZING_CFLAGS) src_test_fuzz_fuzz_microdesc_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_microdesc_LDADD = $(FUZZING_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_fuzz_vrs_SOURCES = \ src/test/fuzz/fuzzing_common.c \ src/test/fuzz/fuzz_vrs.c @@ -141,7 +165,9 @@ src_test_fuzz_fuzz_vrs_CPPFLAGS = $(FUZZING_CPPFLAGS) src_test_fuzz_fuzz_vrs_CFLAGS = $(FUZZING_CFLAGS) src_test_fuzz_fuzz_vrs_LDFLAGS = $(FUZZING_LDFLAG) src_test_fuzz_fuzz_vrs_LDADD = $(FUZZING_LIBS) +endif +if UNITTESTS_ENABLED FUZZERS = \ src/test/fuzz/fuzz-consensus \ src/test/fuzz/fuzz-descriptor \ @@ -155,93 +181,118 @@ FUZZERS = \ src/test/fuzz/fuzz-iptsv2 \ src/test/fuzz/fuzz-microdesc \ src/test/fuzz/fuzz-vrs +endif # ===== libfuzzer if LIBFUZZER_ENABLED +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_consensus_SOURCES = \ $(src_test_fuzz_fuzz_consensus_SOURCES) src_test_fuzz_lf_fuzz_consensus_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) src_test_fuzz_lf_fuzz_consensus_CFLAGS = $(LIBFUZZER_CFLAGS) src_test_fuzz_lf_fuzz_consensus_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_consensus_LDADD = $(LIBFUZZER_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_descriptor_SOURCES = \ $(src_test_fuzz_fuzz_descriptor_SOURCES) src_test_fuzz_lf_fuzz_descriptor_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) src_test_fuzz_lf_fuzz_descriptor_CFLAGS = $(LIBFUZZER_CFLAGS) src_test_fuzz_lf_fuzz_descriptor_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_descriptor_LDADD = $(LIBFUZZER_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_diff_SOURCES = \ $(src_test_fuzz_fuzz_diff_SOURCES) src_test_fuzz_lf_fuzz_diff_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) src_test_fuzz_lf_fuzz_diff_CFLAGS = $(LIBFUZZER_CFLAGS) src_test_fuzz_lf_fuzz_diff_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_diff_LDADD = $(LIBFUZZER_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_diff_apply_SOURCES = \ $(src_test_fuzz_fuzz_diff_apply_SOURCES) src_test_fuzz_lf_fuzz_diff_apply_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) src_test_fuzz_lf_fuzz_diff_apply_CFLAGS = $(LIBFUZZER_CFLAGS) src_test_fuzz_lf_fuzz_diff_apply_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_diff_apply_LDADD = $(LIBFUZZER_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_extrainfo_SOURCES = \ $(src_test_fuzz_fuzz_extrainfo_SOURCES) src_test_fuzz_lf_fuzz_extrainfo_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) src_test_fuzz_lf_fuzz_extrainfo_CFLAGS = $(LIBFUZZER_CFLAGS) src_test_fuzz_lf_fuzz_extrainfo_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_extrainfo_LDADD = $(LIBFUZZER_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_hsdescv2_SOURCES = \ $(src_test_fuzz_fuzz_hsdescv2_SOURCES) src_test_fuzz_lf_fuzz_hsdescv2_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) src_test_fuzz_lf_fuzz_hsdescv2_CFLAGS = $(LIBFUZZER_CFLAGS) src_test_fuzz_lf_fuzz_hsdescv2_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_hsdescv2_LDADD = $(LIBFUZZER_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_hsdescv3_SOURCES = \ $(src_test_fuzz_fuzz_hsdescv3_SOURCES) src_test_fuzz_lf_fuzz_hsdescv3_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) src_test_fuzz_lf_fuzz_hsdescv3_CFLAGS = $(LIBFUZZER_CFLAGS) src_test_fuzz_lf_fuzz_hsdescv3_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_hsdescv3_LDADD = $(LIBFUZZER_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_http_SOURCES = \ $(src_test_fuzz_fuzz_http_SOURCES) src_test_fuzz_lf_fuzz_http_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) src_test_fuzz_lf_fuzz_http_CFLAGS = $(LIBFUZZER_CFLAGS) src_test_fuzz_lf_fuzz_http_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_http_LDADD = $(LIBFUZZER_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_http_connect_SOURCES = \ $(src_test_fuzz_fuzz_http_connect_SOURCES) src_test_fuzz_lf_fuzz_http_connect_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) src_test_fuzz_lf_fuzz_http_connect_CFLAGS = $(LIBFUZZER_CFLAGS) src_test_fuzz_lf_fuzz_http_connect_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_http_connect_LDADD = $(LIBFUZZER_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_iptsv2_SOURCES = \ $(src_test_fuzz_fuzz_iptsv2_SOURCES) src_test_fuzz_lf_fuzz_iptsv2_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) src_test_fuzz_lf_fuzz_iptsv2_CFLAGS = $(LIBFUZZER_CFLAGS) src_test_fuzz_lf_fuzz_iptsv2_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_iptsv2_LDADD = $(LIBFUZZER_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_microdesc_SOURCES = \ $(src_test_fuzz_fuzz_microdesc_SOURCES) src_test_fuzz_lf_fuzz_microdesc_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) src_test_fuzz_lf_fuzz_microdesc_CFLAGS = $(LIBFUZZER_CFLAGS) src_test_fuzz_lf_fuzz_microdesc_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_microdesc_LDADD = $(LIBFUZZER_LIBS) +endif +if UNITTESTS_ENABLED src_test_fuzz_lf_fuzz_vrs_SOURCES = \ $(src_test_fuzz_fuzz_vrs_SOURCES) src_test_fuzz_lf_fuzz_vrs_CPPFLAGS = $(LIBFUZZER_CPPFLAGS) src_test_fuzz_lf_fuzz_vrs_CFLAGS = $(LIBFUZZER_CFLAGS) src_test_fuzz_lf_fuzz_vrs_LDFLAGS = $(LIBFUZZER_LDFLAG) src_test_fuzz_lf_fuzz_vrs_LDADD = $(LIBFUZZER_LIBS) +endif LIBFUZZER_FUZZERS = \ src/test/fuzz/lf-fuzz-consensus \ @@ -264,65 +315,89 @@ endif # ===== oss-fuzz if OSS_FUZZ_ENABLED +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_consensus_a_SOURCES = \ $(src_test_fuzz_fuzz_consensus_SOURCES) src_test_fuzz_liboss_fuzz_consensus_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_consensus_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_descriptor_a_SOURCES = \ $(src_test_fuzz_fuzz_descriptor_SOURCES) src_test_fuzz_liboss_fuzz_descriptor_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_descriptor_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_diff_a_SOURCES = \ $(src_test_fuzz_fuzz_diff_SOURCES) src_test_fuzz_liboss_fuzz_diff_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_diff_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_diff_apply_a_SOURCES = \ $(src_test_fuzz_fuzz_diff_apply_SOURCES) src_test_fuzz_liboss_fuzz_diff_apply_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_diff_apply_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_extrainfo_a_SOURCES = \ $(src_test_fuzz_fuzz_extrainfo_SOURCES) src_test_fuzz_liboss_fuzz_extrainfo_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_extrainfo_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_hsdescv2_a_SOURCES = \ $(src_test_fuzz_fuzz_hsdescv2_SOURCES) src_test_fuzz_liboss_fuzz_hsdescv2_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_hsdescv2_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_hsdescv3_a_SOURCES = \ $(src_test_fuzz_fuzz_hsdescv3_SOURCES) src_test_fuzz_liboss_fuzz_hsdescv3_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_hsdescv3_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_http_a_SOURCES = \ $(src_test_fuzz_fuzz_http_SOURCES) src_test_fuzz_liboss_fuzz_http_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_http_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_http_connect_a_SOURCES = \ $(src_test_fuzz_fuzz_http_connect_SOURCES) src_test_fuzz_liboss_fuzz_http_connect_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_http_connect_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_iptsv2_a_SOURCES = \ $(src_test_fuzz_fuzz_iptsv2_SOURCES) src_test_fuzz_liboss_fuzz_iptsv2_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_iptsv2_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_microdesc_a_SOURCES = \ $(src_test_fuzz_fuzz_microdesc_SOURCES) src_test_fuzz_liboss_fuzz_microdesc_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_microdesc_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif +if UNITTESTS_ENABLED src_test_fuzz_liboss_fuzz_vrs_a_SOURCES = \ $(src_test_fuzz_fuzz_vrs_SOURCES) src_test_fuzz_liboss_fuzz_vrs_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS) src_test_fuzz_liboss_fuzz_vrs_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS) +endif OSS_FUZZ_FUZZERS = \ src/test/fuzz/liboss-fuzz-consensus.a \ diff --git a/src/test/include.am b/src/test/include.am index 93fc008419..70df3aac14 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -10,7 +10,10 @@ TESTS_ENVIRONMENT = \ export TESTING_TOR_BINARY="$(TESTING_TOR_BINARY)"; \ export CARGO="$(CARGO)"; \ export EXTRA_CARGO_OPTIONS="$(EXTRA_CARGO_OPTIONS)"; \ - export CARGO_ONLINE="$(CARGO_ONLINE)"; + export CARGO_ONLINE="$(CARGO_ONLINE)"; \ + export CCLD="$(CCLD)"; \ + chmod +x "$(abs_top_builddir)/link_rust.sh"; \ + export RUSTFLAGS="-C linker=$(abs_top_builddir)/link_rust.sh"; TESTSCRIPTS = \ src/test/fuzz_static_testcases.sh \ @@ -80,7 +83,10 @@ src_test_AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \ # This seems to matter nowhere but on Windows, but I assure you that it # matters a lot there, and is quite hard to debug if you forget to do it. -src_test_test_SOURCES = \ +src_test_test_SOURCES = + +if UNITTESTS_ENABLED +src_test_test_SOURCES += \ src/test/log_test_helpers.c \ src/test/hs_test_helpers.c \ src/test/rend_test_helpers.c \ @@ -89,7 +95,9 @@ src_test_test_SOURCES = \ src/test/test_addr.c \ src/test/test_address.c \ src/test/test_address_set.c \ + src/test/test_bridges.c \ src/test/test_buffers.c \ + src/test/test_bwmgt.c \ src/test/test_cell_formats.c \ src/test/test_cell_queue.c \ src/test/test_channel.c \ @@ -119,6 +127,7 @@ src_test_test_SOURCES = \ src/test/test_dos.c \ src/test/test_entryconn.c \ src/test/test_entrynodes.c \ + src/test/test_geoip.c \ src/test/test_guardfraction.c \ src/test/test_extorport.c \ src/test/test_hs.c \ @@ -137,11 +146,13 @@ src_test_test_SOURCES = \ src/test/test_keypin.c \ src/test/test_link_handshake.c \ src/test/test_logging.c \ + src/test/test_mainloop.c \ src/test/test_microdesc.c \ src/test/test_nodelist.c \ src/test/test_oom.c \ src/test/test_oos.c \ src/test/test_options.c \ + src/test/test_periodic_event.c \ src/test/test_policy.c \ src/test/test_procmon.c \ src/test/test_proto_http.c \ @@ -151,6 +162,7 @@ src_test_test_SOURCES = \ src/test/test_pubsub.c \ src/test/test_relay.c \ src/test/test_relaycell.c \ + src/test/test_relaycrypt.c \ src/test/test_rendcache.c \ src/test/test_replay.c \ src/test/test_router.c \ @@ -167,19 +179,24 @@ src_test_test_SOURCES = \ src/test/test_util.c \ src/test/test_util_format.c \ src/test/test_util_process.c \ + src/test/test_voting_schedule.c \ src/test/test_helpers.c \ src/test/test_dns.c \ src/test/testing_common.c \ src/test/testing_rsakeys.c \ src/ext/tinytest.c +endif -src_test_test_slow_SOURCES = \ +src_test_test_slow_SOURCES = +if UNITTESTS_ENABLED +src_test_test_slow_SOURCES += \ src/test/test_slow.c \ src/test/test_crypto_slow.c \ src/test/test_util_slow.c \ src/test/testing_common.c \ src/test/testing_rsakeys.c \ src/ext/tinytest.c +endif src_test_test_memwipe_SOURCES = \ src/test/test-memwipe.c @@ -326,6 +343,7 @@ src_test_test_hs_ntor_cl_AM_CPPFLAGS = \ -I"$(top_srcdir)/src/or" +if UNITTESTS_ENABLED noinst_PROGRAMS += src/test/test-bt-cl src_test_test_bt_cl_SOURCES = src/test/test_bt_cl.c src_test_test_bt_cl_LDADD = src/common/libor-testing.a \ @@ -333,9 +351,10 @@ src_test_test_bt_cl_LDADD = src/common/libor-testing.a \ src/trace/libor-trace.a \ $(rust_ldadd) \ @TOR_LIB_MATH@ \ - @TOR_LIB_WS32@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ + @TOR_LIB_WS32@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ src_test_test_bt_cl_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) src_test_test_bt_cl_CPPFLAGS= $(src_test_AM_CPPFLAGS) $(TEST_CPPFLAGS) +endif EXTRA_DIST += \ src/test/bt_test.py \ @@ -346,6 +365,7 @@ EXTRA_DIST += \ src/test/fuzz_static_testcases.sh \ src/test/slownacl_curve25519.py \ src/test/zero_length_keys.sh \ + src/test/rust_supp.txt \ src/test/test_keygen.sh \ src/test/test_key_expiration.sh \ src/test/test_zero_length_keys.sh \ diff --git a/src/test/log_test_helpers.h b/src/test/log_test_helpers.h index 70c584eb37..f5bbfcf3ff 100644 --- a/src/test/log_test_helpers.h +++ b/src/test/log_test_helpers.h @@ -85,6 +85,10 @@ void mock_dump_saved_logs(void); assert_log_predicate(!mock_saved_log_has_message(str), \ "expected log to not contain " # str); +#define expect_no_log_msg_containing(str) \ + assert_log_predicate(!mock_saved_log_has_message_containing(str), \ + "expected log to not contain " # str); + #define expect_log_severity(severity) \ assert_log_predicate(mock_saved_log_has_severity(severity), \ "expected log to contain severity " # severity); diff --git a/src/test/rend_test_helpers.c b/src/test/rend_test_helpers.c index 095bfecf21..9ac3894b0b 100644 --- a/src/test/rend_test_helpers.c +++ b/src/test/rend_test_helpers.c @@ -2,6 +2,7 @@ /* See LICENSE for licensing information */ #include "or.h" +#include "crypto_rand.h" #include "test.h" #include "rendcommon.h" #include "rend_test_helpers.h" diff --git a/src/test/rust_supp.txt b/src/test/rust_supp.txt new file mode 100644 index 0000000000..7fa50f3fb1 --- /dev/null +++ b/src/test/rust_supp.txt @@ -0,0 +1 @@ +leak:backtrace_alloc diff --git a/src/test/test-memwipe.c b/src/test/test-memwipe.c index 89d946d506..aaaf2e7f68 100644 --- a/src/test/test-memwipe.c +++ b/src/test/test-memwipe.c @@ -7,7 +7,7 @@ #include <sys/types.h> #include <stdlib.h> -#include "crypto.h" +#include "crypto_util.h" #include "compat.h" #include "util.h" diff --git a/src/test/test-timers.c b/src/test/test-timers.c index a0b5b535c2..f20f29578b 100644 --- a/src/test/test-timers.c +++ b/src/test/test-timers.c @@ -7,11 +7,9 @@ #include <stdio.h> #include <string.h> -#include <event2/event.h> - #include "compat.h" #include "compat_libevent.h" -#include "crypto.h" +#include "crypto_rand.h" #include "timers.h" #include "util.h" @@ -50,7 +48,7 @@ timer_cb(tor_timer_t *t, void *arg, const monotime_t *now_mono) // printf("%d / %d\n",n_fired, N_TIMERS); if (n_fired == n_active_timers) { - event_base_loopbreak(tor_libevent_get_base()); + tor_libevent_exit_loop_after_callback(tor_libevent_get_base()); } } @@ -90,7 +88,7 @@ main(int argc, char **argv) --n_active_timers; } - event_base_loop(tor_libevent_get_base(), 0); + tor_libevent_run_event_loop(tor_libevent_get_base(), 0); int64_t total_difference = 0; uint64_t total_square_difference = 0; diff --git a/src/test/test.c b/src/test/test.c index 2e7d399322..aea3ad6595 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -9,6 +9,7 @@ **/ #include "orconfig.h" +#include "crypto_rand.h" #include <stdio.h> #ifdef HAVE_FCNTL_H @@ -26,7 +27,6 @@ /* These macros pull in declarations for some functions and structures that * are typically file-private. */ -#define GEOIP_PRIVATE #define ROUTER_PRIVATE #define CIRCUITSTATS_PRIVATE #define CIRCUITLIST_PRIVATE @@ -41,7 +41,6 @@ #include "compress.h" #include "config.h" #include "connection_edge.h" -#include "geoip.h" #include "rendcommon.h" #include "rendcache.h" #include "test.h" @@ -345,6 +344,18 @@ test_onion_queues(void *arg) tor_free(onionskin); } +static crypto_cipher_t *crypto_rand_aes_cipher = NULL; + +// Mock replacement for crypto_rand: Generates bytes from a provided AES_CTR +// cipher in <b>crypto_rand_aes_cipher</b>. +static void +crypto_rand_deterministic_aes(char *out, size_t n) +{ + tor_assert(crypto_rand_aes_cipher); + memset(out, 0, n); + crypto_cipher_crypt_inplace(crypto_rand_aes_cipher, out, n); +} + static void test_circuit_timeout(void *arg) { @@ -374,6 +385,11 @@ test_circuit_timeout(void *arg) state = or_state_new(); + // Use a deterministic RNG here, or else we'll get nondeterministic + // coverage in some of the circuitstats functions. + MOCK(crypto_rand, crypto_rand_deterministic_aes); + crypto_rand_aes_cipher = crypto_cipher_new("xyzzyplughplover"); + circuitbuild_running_unit_tests(); #define timeout0 (build_time_t)(30*1000.0) initial.Xm = 3000; @@ -508,6 +524,8 @@ test_circuit_timeout(void *arg) circuit_build_times_free_timeouts(&final); or_state_free(state); teardown_periodic_events(); + UNMOCK(crypto_rand); + crypto_cipher_free(crypto_rand_aes_cipher); } /** Test encoding and parsing of rendezvous service descriptors. */ @@ -623,376 +641,6 @@ test_rend_fns(void *arg) tor_free(intro_points_encrypted); } - /* Record odd numbered fake-IPs using ipv6, even numbered fake-IPs - * using ipv4. Since our fake geoip database is the same between - * ipv4 and ipv6, we should get the same result no matter which - * address family we pick for each IP. */ -#define SET_TEST_ADDRESS(i) do { \ - if ((i) & 1) { \ - SET_TEST_IPV6(i); \ - tor_addr_from_in6(&addr, &in6); \ - } else { \ - tor_addr_from_ipv4h(&addr, (uint32_t) i); \ - } \ - } while (0) - - /* Make sure that country ID actually works. */ -#define SET_TEST_IPV6(i) \ - do { \ - set_uint32(in6.s6_addr + 12, htonl((uint32_t) (i))); \ - } while (0) -#define CHECK_COUNTRY(country, val) do { \ - /* test ipv4 country lookup */ \ - tt_str_op(country, OP_EQ, \ - geoip_get_country_name(geoip_get_country_by_ipv4(val))); \ - /* test ipv6 country lookup */ \ - SET_TEST_IPV6(val); \ - tt_str_op(country, OP_EQ, \ - geoip_get_country_name(geoip_get_country_by_ipv6(&in6))); \ - } while (0) - -/** Run unit tests for GeoIP code. */ -static void -test_geoip(void *arg) -{ - int i, j; - time_t now = 1281533250; /* 2010-08-11 13:27:30 UTC */ - char *s = NULL, *v = NULL; - const char *bridge_stats_1 = - "bridge-stats-end 2010-08-12 13:27:30 (86400 s)\n" - "bridge-ips zz=24,xy=8\n" - "bridge-ip-versions v4=16,v6=16\n" - "bridge-ip-transports <OR>=24\n", - *dirreq_stats_1 = - "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n" - "dirreq-v3-ips ab=8\n" - "dirreq-v3-reqs ab=8\n" - "dirreq-v3-resp ok=0,not-enough-sigs=0,unavailable=0,not-found=0," - "not-modified=0,busy=0\n" - "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n" - "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0\n", - *dirreq_stats_2 = - "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n" - "dirreq-v3-ips \n" - "dirreq-v3-reqs \n" - "dirreq-v3-resp ok=0,not-enough-sigs=0,unavailable=0,not-found=0," - "not-modified=0,busy=0\n" - "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n" - "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0\n", - *dirreq_stats_3 = - "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n" - "dirreq-v3-ips \n" - "dirreq-v3-reqs \n" - "dirreq-v3-resp ok=8,not-enough-sigs=0,unavailable=0,not-found=0," - "not-modified=0,busy=0\n" - "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n" - "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0\n", - *dirreq_stats_4 = - "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n" - "dirreq-v3-ips \n" - "dirreq-v3-reqs \n" - "dirreq-v3-resp ok=8,not-enough-sigs=0,unavailable=0,not-found=0," - "not-modified=0,busy=0\n" - "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n" - "dirreq-v3-tunneled-dl complete=0,timeout=0,running=4\n", - *entry_stats_1 = - "entry-stats-end 2010-08-12 13:27:30 (86400 s)\n" - "entry-ips ab=8\n", - *entry_stats_2 = - "entry-stats-end 2010-08-12 13:27:30 (86400 s)\n" - "entry-ips \n"; - tor_addr_t addr; - struct in6_addr in6; - - /* Populate the DB a bit. Add these in order, since we can't do the final - * 'sort' step. These aren't very good IP addresses, but they're perfectly - * fine uint32_t values. */ - (void)arg; - tt_int_op(0,OP_EQ, geoip_parse_entry("10,50,AB", AF_INET)); - tt_int_op(0,OP_EQ, geoip_parse_entry("52,90,XY", AF_INET)); - tt_int_op(0,OP_EQ, geoip_parse_entry("95,100,AB", AF_INET)); - tt_int_op(0,OP_EQ, geoip_parse_entry("\"105\",\"140\",\"ZZ\"", AF_INET)); - tt_int_op(0,OP_EQ, geoip_parse_entry("\"150\",\"190\",\"XY\"", AF_INET)); - tt_int_op(0,OP_EQ, geoip_parse_entry("\"200\",\"250\",\"AB\"", AF_INET)); - - /* Populate the IPv6 DB equivalently with fake IPs in the same range */ - tt_int_op(0,OP_EQ, geoip_parse_entry("::a,::32,AB", AF_INET6)); - tt_int_op(0,OP_EQ, geoip_parse_entry("::34,::5a,XY", AF_INET6)); - tt_int_op(0,OP_EQ, geoip_parse_entry("::5f,::64,AB", AF_INET6)); - tt_int_op(0,OP_EQ, geoip_parse_entry("::69,::8c,ZZ", AF_INET6)); - tt_int_op(0,OP_EQ, geoip_parse_entry("::96,::be,XY", AF_INET6)); - tt_int_op(0,OP_EQ, geoip_parse_entry("::c8,::fa,AB", AF_INET6)); - - /* We should have 4 countries: ??, ab, xy, zz. */ - tt_int_op(4,OP_EQ, geoip_get_n_countries()); - memset(&in6, 0, sizeof(in6)); - - CHECK_COUNTRY("??", 3); - CHECK_COUNTRY("ab", 32); - CHECK_COUNTRY("??", 5); - CHECK_COUNTRY("??", 51); - CHECK_COUNTRY("xy", 150); - CHECK_COUNTRY("xy", 190); - CHECK_COUNTRY("??", 2000); - - tt_int_op(0,OP_EQ, geoip_get_country_by_ipv4(3)); - SET_TEST_IPV6(3); - tt_int_op(0,OP_EQ, geoip_get_country_by_ipv6(&in6)); - - get_options_mutable()->BridgeRelay = 1; - get_options_mutable()->BridgeRecordUsageByCountry = 1; - /* Put 9 observations in AB... */ - for (i=32; i < 40; ++i) { - SET_TEST_ADDRESS(i); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-7200); - } - SET_TEST_ADDRESS(225); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-7200); - /* and 3 observations in XY, several times. */ - for (j=0; j < 10; ++j) - for (i=52; i < 55; ++i) { - SET_TEST_ADDRESS(i); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-3600); - } - /* and 17 observations in ZZ... */ - for (i=110; i < 127; ++i) { - SET_TEST_ADDRESS(i); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now); - } - geoip_get_client_history(GEOIP_CLIENT_CONNECT, &s, &v); - tt_assert(s); - tt_assert(v); - tt_str_op("zz=24,ab=16,xy=8",OP_EQ, s); - tt_str_op("v4=16,v6=16",OP_EQ, v); - tor_free(s); - tor_free(v); - - /* Now clear out all the AB observations. */ - geoip_remove_old_clients(now-6000); - geoip_get_client_history(GEOIP_CLIENT_CONNECT, &s, &v); - tt_assert(s); - tt_assert(v); - tt_str_op("zz=24,xy=8",OP_EQ, s); - tt_str_op("v4=16,v6=16",OP_EQ, v); - tor_free(s); - tor_free(v); - - /* Start testing bridge statistics by making sure that we don't output - * bridge stats without initializing them. */ - s = geoip_format_bridge_stats(now + 86400); - tt_ptr_op(s, OP_EQ, NULL); - - /* Initialize stats and generate the bridge-stats history string out of - * the connecting clients added above. */ - geoip_bridge_stats_init(now); - s = geoip_format_bridge_stats(now + 86400); - tt_assert(s); - tt_str_op(bridge_stats_1,OP_EQ, s); - tor_free(s); - - /* Stop collecting bridge stats and make sure we don't write a history - * string anymore. */ - geoip_bridge_stats_term(); - s = geoip_format_bridge_stats(now + 86400); - tt_ptr_op(s, OP_EQ, NULL); - - /* Stop being a bridge and start being a directory mirror that gathers - * directory request statistics. */ - geoip_bridge_stats_term(); - get_options_mutable()->BridgeRelay = 0; - get_options_mutable()->BridgeRecordUsageByCountry = 0; - get_options_mutable()->DirReqStatistics = 1; - - /* Start testing dirreq statistics by making sure that we don't collect - * dirreq stats without initializing them. */ - SET_TEST_ADDRESS(100); - geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now); - s = geoip_format_dirreq_stats(now + 86400); - tt_ptr_op(s, OP_EQ, NULL); - - /* Initialize stats, note one connecting client, and generate the - * dirreq-stats history string. */ - geoip_dirreq_stats_init(now); - SET_TEST_ADDRESS(100); - geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now); - s = geoip_format_dirreq_stats(now + 86400); - tt_str_op(dirreq_stats_1,OP_EQ, s); - tor_free(s); - - /* Stop collecting stats, add another connecting client, and ensure we - * don't generate a history string. */ - geoip_dirreq_stats_term(); - SET_TEST_ADDRESS(101); - geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now); - s = geoip_format_dirreq_stats(now + 86400); - tt_ptr_op(s, OP_EQ, NULL); - - /* Re-start stats, add a connecting client, reset stats, and make sure - * that we get an all empty history string. */ - geoip_dirreq_stats_init(now); - SET_TEST_ADDRESS(100); - geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now); - geoip_reset_dirreq_stats(now); - s = geoip_format_dirreq_stats(now + 86400); - tt_str_op(dirreq_stats_2,OP_EQ, s); - tor_free(s); - - /* Note a successful network status response and make sure that it - * appears in the history string. */ - geoip_note_ns_response(GEOIP_SUCCESS); - s = geoip_format_dirreq_stats(now + 86400); - tt_str_op(dirreq_stats_3,OP_EQ, s); - tor_free(s); - - /* Start a tunneled directory request. */ - geoip_start_dirreq((uint64_t) 1, 1024, DIRREQ_TUNNELED); - s = geoip_format_dirreq_stats(now + 86400); - tt_str_op(dirreq_stats_4,OP_EQ, s); - tor_free(s); - - /* Stop collecting directory request statistics and start gathering - * entry stats. */ - geoip_dirreq_stats_term(); - get_options_mutable()->DirReqStatistics = 0; - get_options_mutable()->EntryStatistics = 1; - - /* Start testing entry statistics by making sure that we don't collect - * anything without initializing entry stats. */ - SET_TEST_ADDRESS(100); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now); - s = geoip_format_entry_stats(now + 86400); - tt_ptr_op(s, OP_EQ, NULL); - - /* Initialize stats, note one connecting client, and generate the - * entry-stats history string. */ - geoip_entry_stats_init(now); - SET_TEST_ADDRESS(100); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now); - s = geoip_format_entry_stats(now + 86400); - tt_str_op(entry_stats_1,OP_EQ, s); - tor_free(s); - - /* Stop collecting stats, add another connecting client, and ensure we - * don't generate a history string. */ - geoip_entry_stats_term(); - SET_TEST_ADDRESS(101); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now); - s = geoip_format_entry_stats(now + 86400); - tt_ptr_op(s, OP_EQ, NULL); - - /* Re-start stats, add a connecting client, reset stats, and make sure - * that we get an all empty history string. */ - geoip_entry_stats_init(now); - SET_TEST_ADDRESS(100); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now); - geoip_reset_entry_stats(now); - s = geoip_format_entry_stats(now + 86400); - tt_str_op(entry_stats_2,OP_EQ, s); - tor_free(s); - - /* Test the OOM handler. Add a client, run the OOM. */ - geoip_entry_stats_init(now); - SET_TEST_ADDRESS(100); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, - now - (12 * 60 * 60)); - /* We've seen this 12 hours ago. Run the OOM, it should clean the entry - * because it is above the minimum cutoff of 4 hours. */ - size_t bytes_removed = geoip_client_cache_handle_oom(now, 1000); - tt_size_op(bytes_removed, OP_GT, 0); - - /* Do it again but this time with an entry with a lower cutoff. */ - geoip_entry_stats_init(now); - SET_TEST_ADDRESS(100); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, - now - (3 * 60 * 60)); - bytes_removed = geoip_client_cache_handle_oom(now, 1000); - tt_size_op(bytes_removed, OP_EQ, 0); - - /* Stop collecting entry statistics. */ - geoip_entry_stats_term(); - get_options_mutable()->EntryStatistics = 0; - - done: - tor_free(s); - tor_free(v); -} - -static void -test_geoip_with_pt(void *arg) -{ - time_t now = 1281533250; /* 2010-08-11 13:27:30 UTC */ - char *s = NULL; - int i; - tor_addr_t addr; - struct in6_addr in6; - - (void)arg; - get_options_mutable()->BridgeRelay = 1; - get_options_mutable()->BridgeRecordUsageByCountry = 1; - - memset(&in6, 0, sizeof(in6)); - - /* No clients seen yet. */ - s = geoip_get_transport_history(); - tor_assert(!s); - - /* 4 connections without a pluggable transport */ - for (i=0; i < 4; ++i) { - SET_TEST_ADDRESS(i); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-7200); - } - - /* 9 connections with "alpha" */ - for (i=4; i < 13; ++i) { - SET_TEST_ADDRESS(i); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "alpha", now-7200); - } - - /* one connection with "beta" */ - SET_TEST_ADDRESS(13); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "beta", now-7200); - - /* 14 connections with "charlie" */ - for (i=14; i < 28; ++i) { - SET_TEST_ADDRESS(i); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "charlie", now-7200); - } - - /* 131 connections with "ddr" */ - for (i=28; i < 159; ++i) { - SET_TEST_ADDRESS(i); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "ddr", now-7200); - } - - /* 8 connections with "entropy" */ - for (i=159; i < 167; ++i) { - SET_TEST_ADDRESS(i); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "entropy", now-7200); - } - - /* 2 connections from the same IP with two different transports. */ - SET_TEST_ADDRESS(++i); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "fire", now-7200); - geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "google", now-7200); - - /* Test the transport history string. */ - s = geoip_get_transport_history(); - tor_assert(s); - tt_str_op(s,OP_EQ, "<OR>=8,alpha=16,beta=8,charlie=16,ddr=136," - "entropy=8,fire=8,google=8"); - - /* Stop collecting entry statistics. */ - geoip_entry_stats_term(); - get_options_mutable()->EntryStatistics = 0; - - done: - tor_free(s); -} - -#undef SET_TEST_ADDRESS -#undef SET_TEST_IPV6 -#undef CHECK_COUNTRY - /** Run unit tests for stats code. */ static void test_stats(void *arg) @@ -1166,8 +814,6 @@ static struct testcase_t test_array[] = { { "fast_handshake", test_fast_handshake, 0, NULL, NULL }, FORK(circuit_timeout), FORK(rend_fns), - ENT(geoip), - FORK(geoip_with_pt), FORK(stats), END_OF_TESTCASES @@ -1179,7 +825,9 @@ struct testgroup_t testgroups[] = { { "addr/", addr_tests }, { "address/", address_tests }, { "address_set/", address_set_tests }, + { "bridges/", bridges_tests }, { "buffer/", buffer_tests }, + { "bwmgt/", bwmgt_tests }, { "cellfmt/", cell_format_tests }, { "cellqueue/", cell_queue_tests }, { "channel/", channel_tests }, @@ -1205,11 +853,13 @@ struct testgroup_t testgroups[] = { { "dir/", dir_tests }, { "dir_handle_get/", dir_handle_get_tests }, { "dir/md/", microdesc_tests }, + { "dir/voting-schedule/", voting_schedule_tests }, { "dos/", dos_tests }, { "entryconn/", entryconn_tests }, { "entrynodes/", entrynodes_tests }, { "guardfraction/", guardfraction_tests }, { "extorport/", extorport_tests }, + { "geoip/", geoip_tests }, { "legacy_hs/", hs_tests }, { "hs_cache/", hs_cache }, { "hs_cell/", hs_cell_tests }, @@ -1224,10 +874,12 @@ struct testgroup_t testgroups[] = { { "introduce/", introduce_tests }, { "keypin/", keypin_tests }, { "link-handshake/", link_handshake_tests }, + { "mainloop/", mainloop_tests }, { "nodelist/", nodelist_tests }, { "oom/", oom_tests }, { "oos/", oos_tests }, { "options/", options_tests }, + { "periodic-event/" , periodic_event_tests }, { "policy/" , policy_tests }, { "procmon/", procmon_tests }, { "proto/http/", proto_http_tests }, @@ -1236,6 +888,7 @@ struct testgroup_t testgroups[] = { { "pt/", pt_tests }, { "relay/" , relay_tests }, { "relaycell/", relaycell_tests }, + { "relaycrypt/", relaycrypt_tests }, { "rend_cache/", rend_cache_tests }, { "replaycache/", replaycache_tests }, { "router/", router_tests }, diff --git a/src/test/test.h b/src/test/test.h index 26139fc5fe..63b2b30746 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -11,6 +11,8 @@ * \brief Macros and functions used by unit tests. */ +#define DEBUG_SMARTLIST 1 + #include "compat.h" #include "tinytest.h" #define TT_EXIT_TEST_FUNCTION STMT_BEGIN goto done; STMT_END @@ -72,6 +74,14 @@ I64_PRINTF_TYPE, I64_FORMAT, \ {print_ = (I64_PRINTF_TYPE) value_;}, {}, TT_EXIT_TEST_FUNCTION) +/** + * Declare that the test is done, even though no tt___op() calls were made. + * + * For use when you only want to test calling something, but not check + * any values/pointers/etc afterwards. + */ +#define tt_finished() TT_EXIT_TEST_FUNCTION + const char *get_fname(const char *name); const char *get_fname_rnd(const char *name); struct crypto_pk_t *pk_generate(int idx); @@ -178,6 +188,8 @@ extern struct testcase_t accounting_tests[]; extern struct testcase_t addr_tests[]; extern struct testcase_t address_tests[]; extern struct testcase_t address_set_tests[]; +extern struct testcase_t bridges_tests[]; +extern struct testcase_t bwmgt_tests[]; extern struct testcase_t buffer_tests[]; extern struct testcase_t cell_format_tests[]; extern struct testcase_t cell_queue_tests[]; @@ -208,6 +220,7 @@ extern struct testcase_t entryconn_tests[]; extern struct testcase_t entrynodes_tests[]; extern struct testcase_t guardfraction_tests[]; extern struct testcase_t extorport_tests[]; +extern struct testcase_t geoip_tests[]; extern struct testcase_t hs_tests[]; extern struct testcase_t hs_cache[]; extern struct testcase_t hs_cell_tests[]; @@ -223,11 +236,13 @@ extern struct testcase_t introduce_tests[]; extern struct testcase_t keypin_tests[]; extern struct testcase_t link_handshake_tests[]; extern struct testcase_t logging_tests[]; +extern struct testcase_t mainloop_tests[]; extern struct testcase_t microdesc_tests[]; extern struct testcase_t nodelist_tests[]; extern struct testcase_t oom_tests[]; extern struct testcase_t oos_tests[]; extern struct testcase_t options_tests[]; +extern struct testcase_t periodic_event_tests[]; extern struct testcase_t policy_tests[]; extern struct testcase_t procmon_tests[]; extern struct testcase_t proto_http_tests[]; @@ -237,6 +252,7 @@ extern struct testcase_t pubsub_tests[]; extern struct testcase_t pt_tests[]; extern struct testcase_t relay_tests[]; extern struct testcase_t relaycell_tests[]; +extern struct testcase_t relaycrypt_tests[]; extern struct testcase_t rend_cache_tests[]; extern struct testcase_t replaycache_tests[]; extern struct testcase_t router_tests[]; @@ -252,6 +268,7 @@ extern struct testcase_t tortls_tests[]; extern struct testcase_t util_tests[]; extern struct testcase_t util_format_tests[]; extern struct testcase_t util_process_tests[]; +extern struct testcase_t voting_schedule_tests[]; extern struct testcase_t dns_tests[]; extern struct testcase_t handle_tests[]; extern struct testcase_t sr_tests[]; diff --git a/src/test/test_addr.c b/src/test/test_addr.c index e1a40b7e60..40db31320f 100644 --- a/src/test/test_addr.c +++ b/src/test/test_addr.c @@ -6,8 +6,10 @@ #define ADDRESSMAP_PRIVATE #include "orconfig.h" #include "or.h" +#include "crypto_rand.h" #include "test.h" #include "addressmap.h" +#include "log_test_helpers.h" /** Mocking replacement: only handles localhost. */ static int @@ -941,6 +943,158 @@ test_virtaddrmap(void *data) ; } +static const char *canned_data = NULL; +static size_t canned_data_len = 0; + +/* Mock replacement for crypto_rand() that returns canned data from + * canned_data above. */ +static void +crypto_canned(char *ptr, size_t n) +{ + if (canned_data_len) { + size_t to_copy = MIN(n, canned_data_len); + memcpy(ptr, canned_data, to_copy); + canned_data += to_copy; + canned_data_len -= to_copy; + n -= to_copy; + ptr += to_copy; + } + if (n) { + crypto_rand_unmocked(ptr, n); + } +} + +static void +test_virtaddrmap_persist(void *data) +{ + (void)data; + const char *a, *b, *c; + tor_addr_t addr; + char *ones = NULL; + + addressmap_init(); + + // Try a hostname. + a = addressmap_register_virtual_address(RESOLVED_TYPE_HOSTNAME, + tor_strdup("foobar.baz")); + tt_assert(a); + tt_assert(!strcmpend(a, ".virtual")); + + // mock crypto_rand to repeat the same result twice; make sure we get + // different outcomes. (Because even though the odds for receiving the + // same 80-bit address twice is only 1/2^40, it could still happen for + // some user -- but running our test through 2^40 iterations isn't + // reasonable.) + canned_data = "1234567890" // the first call returns this. + "1234567890" // the second call returns this. + "abcdefghij"; // the third call returns this. + canned_data_len = 30; + MOCK(crypto_rand, crypto_canned); + + a = addressmap_register_virtual_address(RESOLVED_TYPE_HOSTNAME, + tor_strdup("quuxit.baz")); + b = addressmap_register_virtual_address(RESOLVED_TYPE_HOSTNAME, + tor_strdup("nescio.baz")); + tt_assert(a); + tt_assert(b); + tt_str_op(a, OP_EQ, "gezdgnbvgy3tqojq.virtual"); + tt_str_op(b, OP_EQ, "mfrggzdfmztwq2lk.virtual"); + + // Now try something to get us an ipv4 address + UNMOCK(crypto_rand); + tt_int_op(0,OP_EQ, parse_virtual_addr_network("192.168.0.0/16", + AF_INET, 0, NULL)); + a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4, + tor_strdup("foobar.baz")); + tt_assert(a); + tt_assert(!strcmpstart(a, "192.168.")); + tor_addr_parse(&addr, a); + tt_int_op(AF_INET, OP_EQ, tor_addr_family(&addr)); + + b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4, + tor_strdup("quuxit.baz")); + tt_str_op(b, OP_NE, a); + tt_assert(!strcmpstart(b, "192.168.")); + + // Try some canned entropy and verify all the we discard duplicates, + // addresses that end with 0, and addresses that end with 255. + MOCK(crypto_rand, crypto_canned); + canned_data = "\x01\x02\x03\x04" // okay + "\x01\x02\x03\x04" // duplicate + "\x03\x04\x00\x00" // bad ending 1 + "\x05\x05\x00\xff" // bad ending 2 + "\x05\x06\x07\xf0"; // okay + canned_data_len = 20; + a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4, + tor_strdup("wumble.onion")); + b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4, + tor_strdup("wumpus.onion")); + tt_str_op(a, OP_EQ, "192.168.3.4"); + tt_str_op(b, OP_EQ, "192.168.7.240"); + + // Now try IPv6! + UNMOCK(crypto_rand); + tt_int_op(0,OP_EQ, parse_virtual_addr_network("1010:F000::/20", + AF_INET6, 0, NULL)); + a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6, + tor_strdup("foobar.baz")); + tt_assert(a); + tt_assert(!strcmpstart(a, "[1010:f")); + tor_addr_parse(&addr, a); + tt_int_op(AF_INET6, OP_EQ, tor_addr_family(&addr)); + + b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6, + tor_strdup("quuxit.baz")); + tt_str_op(b, OP_NE, a); + tt_assert(!strcmpstart(b, "[1010:f")); + + // Try IPv6 with canned entropy, to make sure we detect duplicates. + MOCK(crypto_rand, crypto_canned); + canned_data = "acanthopterygian" // okay + "cinematographist" // okay + "acanthopterygian" // duplicate + "acanthopterygian" // duplicate + "acanthopterygian" // duplicate + "cinematographist" // duplicate + "coadministration"; // okay + canned_data_len = 16 * 7; + a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6, + tor_strdup("wuffle.baz")); + b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6, + tor_strdup("gribble.baz")); + c = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6, + tor_strdup("surprisingly-legible.baz")); + tt_str_op(a, OP_EQ, "[1010:f16e:7468:6f70:7465:7279:6769:616e]"); + tt_str_op(b, OP_EQ, "[1010:fe65:6d61:746f:6772:6170:6869:7374]"); + tt_str_op(c, OP_EQ, "[1010:f164:6d69:6e69:7374:7261:7469:6f6e]"); + + // Try address exhaustion: make sure we can actually fail if we + // get too many already-existing addresses. + canned_data_len = 128*1024; + canned_data = ones = tor_malloc(canned_data_len); + memset(ones, 1, canned_data_len); + // There is some chance this one will fail if a previous random + // allocation gave out the address already. + a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4, + tor_strdup("might-work.onion")); + if (a) { + tt_str_op(a, OP_EQ, "192.168.1.1"); + } + setup_capture_of_logs(LOG_WARN); + // This one will definitely fail, since we've set up the RNG to hand + // out "1" forever. + b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4, + tor_strdup("wont-work.onion")); + tt_assert(b == NULL); + expect_single_log_msg_containing("Ran out of virtual addresses!"); + + done: + UNMOCK(crypto_rand); + tor_free(ones); + addressmap_free_all(); + teardown_capture_of_logs(); +} + static void test_addr_localname(void *arg) { @@ -1095,6 +1249,7 @@ struct testcase_t addr_tests[] = { ADDR_LEGACY(ip6_helpers), ADDR_LEGACY(parse), { "virtaddr", test_virtaddrmap, 0, NULL, NULL }, + { "virtaddr_persist", test_virtaddrmap_persist, TT_FORK, NULL, NULL }, { "localname", test_addr_localname, 0, NULL, NULL }, { "dup_ip", test_addr_dup_ip, 0, NULL, NULL }, { "sockaddr_to_str", test_addr_sockaddr_to_str, 0, NULL, NULL }, diff --git a/src/test/test_address_set.c b/src/test/test_address_set.c index df022f539a..f7441a6491 100644 --- a/src/test/test_address_set.c +++ b/src/test/test_address_set.c @@ -2,6 +2,7 @@ /* See LICENSE for licensing information */ #include "or.h" +#include "crypto_rand.h" #include "address_set.h" #include "microdesc.h" #include "networkstatus.h" diff --git a/src/test/test_bridges.c b/src/test/test_bridges.c new file mode 100644 index 0000000000..c44f791e0d --- /dev/null +++ b/src/test/test_bridges.c @@ -0,0 +1,614 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_bridges.c + * \brief Unittests for code in src/or/bridges.c + **/ + +#define TOR_BRIDGES_PRIVATE +#define PT_PRIVATE /* Only needed for the mock_* items below */ + +#include <stdbool.h> + +#include "or.h" +#include "address.h" +#include "bridges.h" +#include "config.h" +#include "container.h" +#include "transports.h" +#include "util.h" + +/* Test suite stuff */ +#include "test.h" + +/** + * A mocked transport_t, constructed via mock_transport_get_by_name(). + */ +static transport_t *mock_transport = NULL; + +/** + * Mock transport_get_by_name() to simply return a transport_t for the + * transport name that was input to it. + */ +static transport_t * +mock_transport_get_by_name(const char *name) +{ + tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t)); + uint16_t port = 9999; + int socksv = 9; + char *args = tor_strdup("foo=bar"); + + if (!mock_transport) { + tor_addr_parse(addr, "99.99.99.99"); + mock_transport = transport_new(addr, port, name, socksv, args); + } + + tor_free(addr); + tor_free(args); + + return mock_transport; +} + +#undef PT_PRIVATE /* defined(PT_PRIVATE) */ + +/** + * Test helper: Add a variety of bridges to our global bridgelist. + */ +static void +helper_add_bridges_to_bridgelist(void *arg) +{ + /* Note: the two bridges which do not have specified fingerprints will be + * internally stored as both having the same fingerprint of all-zero bytes. + */ + + (void)arg; + char *bridge0 = tor_strdup("6.6.6.6:6666"); + char *bridge1 = tor_strdup("6.6.6.7:6667 " + "A10C4F666D27364036B562823E5830BC448E046A"); + char *bridge2 = tor_strdup("obfs4 198.245.60.51:443 " + "752CF7825B3B9EA6A98C83AC41F7099D67007EA5 " + "cert=xpmQtKUqQ/6v5X7ijgYE/f03+l2/EuQ1dexjyUhh16wQlu/" + "cpXUGalmhDIlhuiQPNEKmKw iat-mode=0"); + char *bridge3 = tor_strdup("banana 5.5.5.5:5555 " + "9D6AE1BD4FDF39721CE908966E79E16F9BFCCF2F"); + char *bridge4 = tor_strdup("obfs4 1.2.3.4:1234 " + "foo=abcdefghijklmnopqrstuvwxyz"); + char *bridge5 = tor_strdup("apple 4.4.4.4:4444 " + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA " + "foo=abcdefghijklmnopqrstuvwxyz"); + + mark_bridge_list(); + +#define ADD_BRIDGE(bridge) \ + bridge_line_t *bridge_line_ ##bridge = parse_bridge_line(bridge); \ + if (!bridge_line_ ##bridge) { \ + printf("Unparseable bridge line: '%s'", #bridge); \ + } else { \ + bridge_add_from_config(bridge_line_ ##bridge); \ + } \ + tor_free(bridge); + + ADD_BRIDGE(bridge0); + ADD_BRIDGE(bridge1); + ADD_BRIDGE(bridge2); + ADD_BRIDGE(bridge3); + ADD_BRIDGE(bridge4); + ADD_BRIDGE(bridge5); +#undef ADD_BRIDGES + + sweep_bridge_list(); +} + +/** + * Make sure our test helper works too. + */ +static void +test_bridges_helper_func_add_bridges_to_bridgelist(void *arg) +{ + helper_add_bridges_to_bridgelist(arg); + tt_finished(); + + done: + mark_bridge_list(); + sweep_bridge_list(); +} + +/** + * Calling bridge_list_get() should create a new bridgelist if we + * didn't have one before. + */ +static void +test_bridges_bridge_list_get_creates_new_bridgelist(void *arg) +{ + const smartlist_t *bridgelist = bridge_list_get(); + + (void)arg; + + tt_ptr_op(bridgelist, OP_NE, NULL); + + done: + return; +} + +/** + * Calling clear_bridge_list() should remove all bridges from the bridgelist. + */ +static void +test_bridges_clear_bridge_list(void *arg) +{ + const smartlist_t *bridgelist; + const smartlist_t *bridgelist_after; + const bridge_info_t *bridge; + + helper_add_bridges_to_bridgelist(arg); + bridgelist = bridge_list_get(); + tt_ptr_op(bridgelist, OP_NE, NULL); + + bridge = smartlist_get(bridgelist, 0); + tt_ptr_op(bridge, OP_NE, NULL); + + clear_bridge_list(); + bridgelist_after = bridge_list_get(); + tt_ptr_op(bridgelist_after, OP_NE, NULL); + tt_int_op(smartlist_len(bridgelist_after), OP_EQ, 0); + + done: + return; +} + +/** + * Calling bridge_get_addrport() should give me the address and port + * of the bridge. In this case, we sort the smartlist of bridges on + * fingerprints and choose the first one. + */ +static void +test_bridges_bridge_get_addrport(void *arg) +{ + smartlist_t *bridgelist; + const bridge_info_t *bridge; + const tor_addr_port_t *addrport; + + helper_add_bridges_to_bridgelist(arg); + bridgelist = (smartlist_t*)bridge_list_get(); + tt_ptr_op(bridgelist, OP_NE, NULL); + + // This should be the bridge at 6.6.6.6:6666 with fingerprint + // 0000000000000000000000000000000000000000 + bridge = smartlist_get(bridgelist, 0); + tt_ptr_op(bridge, OP_NE, NULL); + + addrport = bridge_get_addr_port(bridge); + tt_int_op(addrport->port, OP_EQ, 6666); + + done: + mark_bridge_list(); + sweep_bridge_list(); +} + +/** + * Calling get_configured_bridge_by_orports_digest() with two + * configured bridge orports and an invalid digest should return the + * bridge of the first addrport in the list. + */ +static void +test_bridges_get_configured_bridge_by_orports_digest(void *arg) +{ + smartlist_t *orports = NULL; + const smartlist_t *bridgelist; + const bridge_info_t *bridge1; + const bridge_info_t *bridge2; + const bridge_info_t *ret; + tor_addr_port_t *addrport1; + tor_addr_port_t *addrport2; + const char *digest; + + helper_add_bridges_to_bridgelist(arg); + bridgelist = bridge_list_get(); + tt_ptr_op(bridgelist, OP_NE, NULL); + + // This should be the bridge at 6.6.6.6:6666 with fingerprint + // 0000000000000000000000000000000000000000 + bridge1 = smartlist_get(bridgelist, 0); + tt_ptr_op(bridge1, OP_NE, NULL); + // This should be the bridge at 6.6.6.7:6667 with fingerprint + // A10C4F666D27364036B562823E5830BC448E046A + bridge2 = smartlist_get(bridgelist, 1); + tt_ptr_op(bridge2, OP_NE, NULL); + + addrport1 = (tor_addr_port_t*)bridge_get_addr_port(bridge1); + tt_int_op(addrport1->port, OP_EQ, 6666); + addrport2 = (tor_addr_port_t*)bridge_get_addr_port(bridge2); + tt_int_op(addrport2->port, OP_EQ, 6667); + + orports = smartlist_new(); + smartlist_add(orports, addrport1); + smartlist_add(orports, addrport2); + + digest = "zzzzzzzzzzzzzzzz"; + + ret = get_configured_bridge_by_orports_digest(digest, orports); + tt_ptr_op(ret, OP_NE, NULL); + + tt_assert(tor_addr_port_eq(addrport1, bridge_get_addr_port(ret))); + + done: + smartlist_free(orports); + + mark_bridge_list(); + sweep_bridge_list(); +} + +/** + * Calling get_configured_bridge_by_addr_port_digest() with a digest that we do + * have and an addr:port pair we don't should return the bridge for that + * digest. + */ +static void +test_bridges_get_configured_bridge_by_addr_port_digest_digest_only(void *arg) +{ + char digest[DIGEST_LEN]; + bridge_info_t *bridge; + const char fingerprint[HEX_DIGEST_LEN] = + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t)); + char ret_addr[16]; + uint16_t port = 11111; + int ret; + + helper_add_bridges_to_bridgelist(arg); + + // We don't actually have a bridge with this addr:port pair + base16_decode(digest, DIGEST_LEN, fingerprint, HEX_DIGEST_LEN); + ret = tor_addr_parse(addr, "111.111.111.111"); + tt_int_op(ret, OP_EQ, 2); // it returns the address family on success + + bridge = get_configured_bridge_by_addr_port_digest(addr, port, digest); + tt_ptr_op(bridge, OP_NE, NULL); + + tor_addr_to_str(ret_addr, &bridge_get_addr_port(bridge)->addr, 16, 0); + tt_str_op("4.4.4.4", OP_EQ, ret_addr); + + done: + tor_free(addr); + + mark_bridge_list(); + sweep_bridge_list(); +} + +/** + * Calling get_configured_bridge_by_addr_port_digest() with only an + * addr:port (i.e. digest set to NULL) should return the bridge for + * that digest when there is such a bridge. + */ +static void +test_bridges_get_configured_bridge_by_addr_port_digest_address_only(void *arg) +{ + bridge_info_t *bridge; + tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t)); + char ret_addr[16]; + uint16_t port = 6666; + int ret; + + helper_add_bridges_to_bridgelist(arg); + + ret = tor_addr_parse(addr, "6.6.6.6"); + tt_int_op(ret, OP_EQ, 2); // it returns the address family on success + + bridge = get_configured_bridge_by_addr_port_digest(addr, port, NULL); + tt_ptr_op(bridge, OP_NE, NULL); + + tor_addr_to_str(ret_addr, &bridge_get_addr_port(bridge)->addr, 16, 0); + tt_str_op("6.6.6.6", OP_EQ, ret_addr); + + done: + tor_free(addr); + + mark_bridge_list(); + sweep_bridge_list(); +} + +/** + * Calling get_configured_bridge_by_exact_addr_port_digest() with a digest that + * we do have, and an addr:port pair we don't have, should return NULL. + */ +static void +test_bridges_get_configured_bridge_by_exact_addr_port_digest_donly(void *arg) +{ + char digest[DIGEST_LEN]; + bridge_info_t *bridge; + const char fingerprint[HEX_DIGEST_LEN] = + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t)); + uint16_t port = 11111; + int ret; + + helper_add_bridges_to_bridgelist(arg); + + // We don't actually have a bridge with this addr:port pair + base16_decode(digest, DIGEST_LEN, fingerprint, HEX_DIGEST_LEN); + ret = tor_addr_parse(addr, "111.111.111.111"); + tt_int_op(ret, OP_EQ, 2); // it returns the address family on success + + bridge = get_configured_bridge_by_exact_addr_port_digest(addr, port, digest); + tt_ptr_op(bridge, OP_EQ, NULL); + + done: + tor_free(addr); + + mark_bridge_list(); + sweep_bridge_list(); +} + +/** + * Calling get_configured_bridge_by_exact_addr_port_digest() with a digest that + * we do have, and an addr:port pair we do have, should return the bridge. + */ +static void +test_bridges_get_configured_bridge_by_exact_addr_port_digest_both(void *arg) +{ + char digest[DIGEST_LEN]; + bridge_info_t *bridge; + const char fingerprint[HEX_DIGEST_LEN] = + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t)); + uint16_t port = 4444; + char ret_addr[16]; + int ret; + + helper_add_bridges_to_bridgelist(arg); + + base16_decode(digest, DIGEST_LEN, fingerprint, HEX_DIGEST_LEN); + ret = tor_addr_parse(addr, "4.4.4.4"); + tt_int_op(ret, OP_EQ, 2); // it returns the address family on success + + bridge = get_configured_bridge_by_exact_addr_port_digest(addr, port, digest); + tt_ptr_op(bridge, OP_NE, NULL); + + tor_addr_to_str(ret_addr, &bridge_get_addr_port(bridge)->addr, 16, 0); + tt_str_op("4.4.4.4", OP_EQ, ret_addr); + + done: + tor_free(addr); + + mark_bridge_list(); + sweep_bridge_list(); +} + +/** + * Calling get_configured_bridge_by_exact_addr_port_digest() with no digest, + * and an addr:port pair we do have, should return the bridge. + */ +static void +test_bridges_get_configured_bridge_by_exact_addr_port_digest_aonly(void *arg) +{ + bridge_info_t *bridge; + tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t)); + uint16_t port = 4444; + char ret_addr[16]; + int ret; + + helper_add_bridges_to_bridgelist(arg); + + ret = tor_addr_parse(addr, "4.4.4.4"); + tt_int_op(ret, OP_EQ, 2); // it returns the address family on success + + bridge = get_configured_bridge_by_exact_addr_port_digest(addr, port, NULL); + tt_ptr_op(bridge, OP_NE, NULL); + + tor_addr_to_str(ret_addr, &bridge_get_addr_port(bridge)->addr, 16, 0); + tt_str_op("4.4.4.4", OP_EQ, ret_addr); + + done: + tor_free(addr); + + mark_bridge_list(); + sweep_bridge_list(); +} + +/** + * Calling find_bridge_by_digest() when we have a bridge with a known + * identity digest should return the bridge's information. + */ +static void +test_bridges_find_bridge_by_digest_known(void *arg) +{ + char digest1[DIGEST_LEN]; + bridge_info_t *bridge; + const char fingerprint[HEX_DIGEST_LEN] = + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + + helper_add_bridges_to_bridgelist(arg); + + base16_decode(digest1, DIGEST_LEN, fingerprint, HEX_DIGEST_LEN); + bridge = find_bridge_by_digest(digest1); + + tt_ptr_op(bridge, OP_NE, NULL); + + /* We have to call bridge_get_rsa_id_digest() here because the bridge_info_t + * struct is opaquely defined in bridges.h. */ + const uint8_t *digest2 = bridge_get_rsa_id_digest(bridge); + + tt_mem_op((char*)digest2, OP_EQ, digest1, DIGEST_LEN); + + done: + mark_bridge_list(); + sweep_bridge_list(); +} + +/** + * Calling find_bridge_by_digest() when we do NOT have a bridge with that + * identity digest should return NULL. + */ +static void +test_bridges_find_bridge_by_digest_unknown(void *arg) +{ + const char *fingerprint = "cccccccccccccccccccccccccccccccccccccccc"; + bridge_info_t *bridge; + + helper_add_bridges_to_bridgelist(arg); + + bridge = find_bridge_by_digest(fingerprint); + + tt_ptr_op(bridge, OP_EQ, NULL); + + done: + mark_bridge_list(); + sweep_bridge_list(); +} + +/** + * Calling bridge_resolve_conflicts() with an identical bridge to one we've + * already configure should mark the pre-configured bridge for removal. + */ +static void +test_bridges_bridge_resolve_conflicts(void *arg) +{ + tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t)); + uint16_t port = 4444; + const char *digest = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + const char *transport = "apple"; + int ret; + + helper_add_bridges_to_bridgelist(arg); + + ret = tor_addr_parse(addr, "4.4.4.4"); + tt_int_op(ret, OP_EQ, 2); // it returns the address family on success + + bridge_resolve_conflicts((const tor_addr_t*)addr, port, digest, transport); + + /* The bridge should now be marked for removal, and removed when we sweep the + * bridge_list */ + sweep_bridge_list(); + ret = addr_is_a_configured_bridge((const tor_addr_t*)addr, port, digest); + tt_int_op(ret, OP_EQ, 0); + + done: + tor_free(addr); + + mark_bridge_list(); + sweep_bridge_list(); +} + +/** + * Calling transport_is_needed() with a transport we do need ("obfs4") and a + * bogus transport that we don't need should return 1 and 0, respectively. + */ +static void +test_bridges_transport_is_needed(void *arg) +{ + int ret; + + helper_add_bridges_to_bridgelist(arg); + + ret = transport_is_needed("obfs4"); + tt_int_op(ret, OP_EQ, 1); + + ret = transport_is_needed("apowefjaoewpaief"); + tt_int_op(ret, OP_EQ, 0); + + done: + mark_bridge_list(); + sweep_bridge_list(); +} + +/** + * Calling get_transport_by_bridge_addrport() with the address and port of a + * configured bridge which uses a pluggable transport when there is no global + * transport_list should return -1 and the transport_t should be NULL. + */ +static void +test_bridges_get_transport_by_bridge_addrport_no_ptlist(void *arg) +{ + transport_t *transport = NULL; + tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t)); + uint16_t port = 1234; + int ret; + + helper_add_bridges_to_bridgelist(arg); + + ret = tor_addr_parse(addr, "1.2.3.4"); + tt_int_op(ret, OP_EQ, 2); // it returns the address family on success? + + /* This will fail because the global transport_list has nothing in it, and so + * transport_get_by_name() has nothing to return, even the the bridge *did* + * say it had an obfs4 transport. + */ + ret = get_transport_by_bridge_addrport((const tor_addr_t*)addr, port, + (const transport_t**)&transport); + tt_int_op(ret, OP_EQ, -1); // returns -1 on failure + tt_ptr_op(transport, OP_EQ, NULL); + + done: + tor_free(addr); + + mark_bridge_list(); + sweep_bridge_list(); +} + +#define PT_PRIVATE + +/** + * Calling get_transport_by_bridge_addrport() with the address and port of a + * configured bridge which uses a pluggable transport should return 0 and set + * appropriate transport_t. + */ +static void +test_bridges_get_transport_by_bridge_addrport(void *arg) +{ + transport_t *transport = NULL; + tor_addr_t *addr = tor_malloc(sizeof(tor_addr_t)); + uint16_t port = 1234; + int ret; + + helper_add_bridges_to_bridgelist(arg); + mark_transport_list(); // Also initialise our transport_list + + ret = tor_addr_parse(addr, "1.2.3.4"); + tt_int_op(ret, OP_EQ, 2); // it returns the address family on success? + + /* After we mock transport_get_by_name() to return a bogus transport_t with + * the name it was asked for, the call should succeed. + */ + MOCK(transport_get_by_name, mock_transport_get_by_name); + ret = get_transport_by_bridge_addrport((const tor_addr_t*)addr, port, + (const transport_t**)&transport); + tt_int_op(ret, OP_EQ, 0); // returns 0 on success + tt_ptr_op(transport, OP_NE, NULL); + tt_str_op(transport->name, OP_EQ, "obfs4"); + + done: + UNMOCK(transport_get_by_name); + + tor_free(addr); + transport_free(transport); + + mark_bridge_list(); + sweep_bridge_list(); +} + +#undef PT_PRIVATE /* defined(PT_PRIVATE) */ + +#define B_TEST(name, flags) \ + { #name, test_bridges_ ##name, (flags), NULL, NULL } + +struct testcase_t bridges_tests[] = { + B_TEST(helper_func_add_bridges_to_bridgelist, 0), + B_TEST(bridge_list_get_creates_new_bridgelist, 0), + B_TEST(clear_bridge_list, 0), + B_TEST(bridge_get_addrport, 0), + B_TEST(get_configured_bridge_by_orports_digest, 0), + B_TEST(get_configured_bridge_by_addr_port_digest_digest_only, 0), + B_TEST(get_configured_bridge_by_addr_port_digest_address_only, 0), + B_TEST(get_configured_bridge_by_exact_addr_port_digest_donly, 0), + B_TEST(get_configured_bridge_by_exact_addr_port_digest_both, 0), + B_TEST(get_configured_bridge_by_exact_addr_port_digest_aonly, 0), + B_TEST(find_bridge_by_digest_known, 0), + B_TEST(find_bridge_by_digest_unknown, 0), + B_TEST(bridge_resolve_conflicts, 0), + B_TEST(get_transport_by_bridge_addrport_no_ptlist, 0), + B_TEST(get_transport_by_bridge_addrport, 0), + B_TEST(transport_is_needed, 0), + END_OF_TESTCASES +}; + diff --git a/src/test/test_buffers.c b/src/test/test_buffers.c index 057d9fa2dc..868f6a8ba4 100644 --- a/src/test/test_buffers.c +++ b/src/test/test_buffers.c @@ -8,6 +8,7 @@ #include "or.h" #include "buffers.h" #include "buffers_tls.h" +#include "crypto_rand.h" #include "proto_http.h" #include "proto_socks.h" #include "test.h" diff --git a/src/test/test_bwmgt.c b/src/test/test_bwmgt.c new file mode 100644 index 0000000000..463a36b24e --- /dev/null +++ b/src/test/test_bwmgt.c @@ -0,0 +1,233 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_bwmgt.c + * \brief tests for bandwidth management / token bucket functions + */ + +#define TOKEN_BUCKET_PRIVATE + +#include "or.h" +#include "test.h" + +#include "token_bucket.h" + +// an imaginary time, in timestamp units. Chosen so it will roll over. +static const uint32_t START_TS = UINT32_MAX-10; +static const int32_t KB = 1024; +static const uint32_t GB = (U64_LITERAL(1) << 30); + +static void +test_bwmgt_token_buf_init(void *arg) +{ + (void)arg; + token_bucket_rw_t b; + + token_bucket_rw_init(&b, 16*KB, 64*KB, START_TS); + // Burst is correct + tt_uint_op(b.cfg.burst, OP_EQ, 64*KB); + // Rate is correct, within 1 percent. + { + uint32_t ticks_per_sec = + (uint32_t) monotime_msec_to_approx_coarse_stamp_units(1000); + uint32_t rate_per_sec = (b.cfg.rate * ticks_per_sec / TICKS_PER_STEP); + + tt_uint_op(rate_per_sec, OP_GT, 16*KB-160); + tt_uint_op(rate_per_sec, OP_LT, 16*KB+160); + } + // Bucket starts out full: + tt_uint_op(b.last_refilled_at_timestamp, OP_EQ, START_TS); + tt_int_op(b.read_bucket.bucket, OP_EQ, 64*KB); + + done: + ; +} + +static void +test_bwmgt_token_buf_adjust(void *arg) +{ + (void)arg; + token_bucket_rw_t b; + + token_bucket_rw_init(&b, 16*KB, 64*KB, START_TS); + + uint32_t rate_orig = b.cfg.rate; + // Increasing burst + token_bucket_rw_adjust(&b, 16*KB, 128*KB); + tt_uint_op(b.cfg.rate, OP_EQ, rate_orig); + tt_uint_op(b.read_bucket.bucket, OP_EQ, 64*KB); + tt_uint_op(b.cfg.burst, OP_EQ, 128*KB); + + // Decreasing burst but staying above bucket + token_bucket_rw_adjust(&b, 16*KB, 96*KB); + tt_uint_op(b.cfg.rate, OP_EQ, rate_orig); + tt_uint_op(b.read_bucket.bucket, OP_EQ, 64*KB); + tt_uint_op(b.cfg.burst, OP_EQ, 96*KB); + + // Decreasing burst below bucket, + token_bucket_rw_adjust(&b, 16*KB, 48*KB); + tt_uint_op(b.cfg.rate, OP_EQ, rate_orig); + tt_uint_op(b.read_bucket.bucket, OP_EQ, 48*KB); + tt_uint_op(b.cfg.burst, OP_EQ, 48*KB); + + // Changing rate. + token_bucket_rw_adjust(&b, 32*KB, 48*KB); + tt_uint_op(b.cfg.rate, OP_GE, rate_orig*2 - 10); + tt_uint_op(b.cfg.rate, OP_LE, rate_orig*2 + 10); + tt_uint_op(b.read_bucket.bucket, OP_EQ, 48*KB); + tt_uint_op(b.cfg.burst, OP_EQ, 48*KB); + + done: + ; +} + +static void +test_bwmgt_token_buf_dec(void *arg) +{ + (void)arg; + token_bucket_rw_t b; + token_bucket_rw_init(&b, 16*KB, 64*KB, START_TS); + + // full-to-not-full. + tt_int_op(0, OP_EQ, token_bucket_rw_dec_read(&b, KB)); + tt_int_op(b.read_bucket.bucket, OP_EQ, 63*KB); + + // Full to almost-not-full + tt_int_op(0, OP_EQ, token_bucket_rw_dec_read(&b, 63*KB - 1)); + tt_int_op(b.read_bucket.bucket, OP_EQ, 1); + + // almost-not-full to empty. + tt_int_op(1, OP_EQ, token_bucket_rw_dec_read(&b, 1)); + tt_int_op(b.read_bucket.bucket, OP_EQ, 0); + + // reset bucket, try full-to-empty + token_bucket_rw_init(&b, 16*KB, 64*KB, START_TS); + tt_int_op(1, OP_EQ, token_bucket_rw_dec_read(&b, 64*KB)); + tt_int_op(b.read_bucket.bucket, OP_EQ, 0); + + // reset bucket, try underflow. + token_bucket_rw_init(&b, 16*KB, 64*KB, START_TS); + tt_int_op(1, OP_EQ, token_bucket_rw_dec_read(&b, 64*KB + 1)); + tt_int_op(b.read_bucket.bucket, OP_EQ, -1); + + // A second underflow does not make the bucket empty. + tt_int_op(0, OP_EQ, token_bucket_rw_dec_read(&b, 1000)); + tt_int_op(b.read_bucket.bucket, OP_EQ, -1001); + + done: + ; +} + +static void +test_bwmgt_token_buf_refill(void *arg) +{ + (void)arg; + token_bucket_rw_t b; + const uint32_t BW_SEC = + (uint32_t)monotime_msec_to_approx_coarse_stamp_units(1000); + token_bucket_rw_init(&b, 16*KB, 64*KB, START_TS); + + /* Make the buffer much emptier, then let one second elapse. */ + token_bucket_rw_dec_read(&b, 48*KB); + tt_int_op(b.read_bucket.bucket, OP_EQ, 16*KB); + tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + BW_SEC)); + tt_int_op(b.read_bucket.bucket, OP_GT, 32*KB - 300); + tt_int_op(b.read_bucket.bucket, OP_LT, 32*KB + 300); + + /* Another half second. */ + tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + BW_SEC*3/2)); + tt_int_op(b.read_bucket.bucket, OP_GT, 40*KB - 400); + tt_int_op(b.read_bucket.bucket, OP_LT, 40*KB + 400); + tt_uint_op(b.last_refilled_at_timestamp, OP_EQ, START_TS + BW_SEC*3/2); + + /* No time: nothing happens. */ + { + const uint32_t bucket_orig = b.read_bucket.bucket; + tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + BW_SEC*3/2)); + tt_int_op(b.read_bucket.bucket, OP_EQ, bucket_orig); + } + + /* Another 30 seconds: fill the bucket. */ + tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, + START_TS + BW_SEC*3/2 + BW_SEC*30)); + tt_int_op(b.read_bucket.bucket, OP_EQ, b.cfg.burst); + tt_uint_op(b.last_refilled_at_timestamp, OP_EQ, + START_TS + BW_SEC*3/2 + BW_SEC*30); + + /* Another 30 seconds: nothing happens. */ + tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, + START_TS + BW_SEC*3/2 + BW_SEC*60)); + tt_int_op(b.read_bucket.bucket, OP_EQ, b.cfg.burst); + tt_uint_op(b.last_refilled_at_timestamp, OP_EQ, + START_TS + BW_SEC*3/2 + BW_SEC*60); + + /* Empty the bucket, let two seconds pass, and make sure that a refill is + * noticed. */ + tt_int_op(1, OP_EQ, token_bucket_rw_dec_read(&b, b.cfg.burst)); + tt_int_op(0, OP_EQ, b.read_bucket.bucket); + tt_int_op(1, OP_EQ, token_bucket_rw_refill(&b, + START_TS + BW_SEC*3/2 + BW_SEC*61)); + tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, + START_TS + BW_SEC*3/2 + BW_SEC*62)); + tt_int_op(b.read_bucket.bucket, OP_GT, 32*KB-400); + tt_int_op(b.read_bucket.bucket, OP_LT, 32*KB+400); + + /* Underflow the bucket, make sure we detect when it has tokens again. */ + tt_int_op(1, OP_EQ, + token_bucket_rw_dec_read(&b, b.read_bucket.bucket+16*KB)); + tt_int_op(-16*KB, OP_EQ, b.read_bucket.bucket); + // half a second passes... + tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + BW_SEC*64)); + tt_int_op(b.read_bucket.bucket, OP_GT, -8*KB-300); + tt_int_op(b.read_bucket.bucket, OP_LT, -8*KB+300); + // a second passes + tt_int_op(1, OP_EQ, token_bucket_rw_refill(&b, START_TS + BW_SEC*65)); + tt_int_op(b.read_bucket.bucket, OP_GT, 8*KB-400); + tt_int_op(b.read_bucket.bucket, OP_LT, 8*KB+400); + + // We step a second backwards, and nothing happens. + tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, START_TS + BW_SEC*64)); + tt_int_op(b.read_bucket.bucket, OP_GT, 8*KB-400); + tt_int_op(b.read_bucket.bucket, OP_LT, 8*KB+400); + + // A ridiculous amount of time passes. + tt_int_op(0, OP_EQ, token_bucket_rw_refill(&b, INT32_MAX)); + tt_int_op(b.read_bucket.bucket, OP_EQ, b.cfg.burst); + + done: + ; +} + +/* Test some helper functions we use within the token bucket interface. */ +static void +test_bwmgt_token_buf_helpers(void *arg) +{ + uint32_t ret; + + (void) arg; + + /* The returned value will be OS specific but in any case, it should be + * greater than 1 since we are passing 1GB/sec rate. */ + ret = rate_per_sec_to_rate_per_step(1 * GB); + tt_u64_op(ret, OP_GT, 1); + + /* We default to 1 in case rate is 0. */ + ret = rate_per_sec_to_rate_per_step(0); + tt_u64_op(ret, OP_EQ, 1); + + done: + ; +} + +#define BWMGT(name) \ + { #name, test_bwmgt_ ## name , 0, NULL, NULL } + +struct testcase_t bwmgt_tests[] = { + BWMGT(token_buf_init), + BWMGT(token_buf_adjust), + BWMGT(token_buf_dec), + BWMGT(token_buf_refill), + BWMGT(token_buf_helpers), + END_OF_TESTCASES +}; diff --git a/src/test/test_cell_formats.c b/src/test/test_cell_formats.c index 88cdef383f..54d9716780 100644 --- a/src/test/test_cell_formats.c +++ b/src/test/test_cell_formats.c @@ -12,6 +12,7 @@ #include "connection_edge.h" #include "connection_or.h" #include "config.h" +#include "crypto_rand.h" #include "onion.h" #include "onion_tap.h" #include "onion_fast.h" diff --git a/src/test/test_channel.c b/src/test/test_channel.c index bdc9d32f78..76124a6e75 100644 --- a/src/test/test_channel.c +++ b/src/test/test_channel.c @@ -12,6 +12,7 @@ #include "circuitmux_ewma.h" /* For var_cell_free */ #include "connection_or.h" +#include "crypto_rand.h" /* For packed_cell stuff */ #define RELAY_PRIVATE #include "relay.h" @@ -281,6 +282,7 @@ new_fake_channel(void) chan->state = CHANNEL_STATE_OPEN; chan->cmux = circuitmux_alloc(); + circuitmux_set_policy(chan->cmux, &ewma_policy); return chan; } @@ -543,6 +545,13 @@ test_channel_outbound_cell(void *arg) (void) arg; + /* Set the test time to be mocked, since this test assumes that no + * time will pass, ewma values will not need to be re-scaled, and so on */ + monotime_enable_test_mocking(); + monotime_set_mock_time_nsec(U64_LITERAL(1000000000) * 12345); + + cmux_ewma_set_options(NULL,NULL); + /* The channel will be freed so we need to hijack this so the scheduler * doesn't get confused. */ MOCK(scheduler_release_channel, scheduler_release_channel_mock); @@ -575,15 +584,13 @@ test_channel_outbound_cell(void *arg) channel_register(chan); tt_int_op(chan->registered, OP_EQ, 1); /* Set EWMA policy so we can pick it when flushing. */ - channel_set_cmux_policy_everywhere(&ewma_policy); + circuitmux_set_policy(chan->cmux, &ewma_policy); tt_ptr_op(circuitmux_get_policy(chan->cmux), OP_EQ, &ewma_policy); /* Register circuit to the channel circid map which will attach the circuit * to the channel's cmux as well. */ circuit_set_n_circid_chan(TO_CIRCUIT(circ), 42, chan); tt_int_op(channel_num_circuits(chan), OP_EQ, 1); - tt_assert(!TO_CIRCUIT(circ)->next_active_on_n_chan); - tt_assert(!TO_CIRCUIT(circ)->prev_active_on_n_chan); /* Test the cmux state. */ tt_ptr_op(TO_CIRCUIT(circ)->n_mux, OP_EQ, chan->cmux); tt_int_op(circuitmux_is_circuit_attached(chan->cmux, TO_CIRCUIT(circ)), @@ -659,6 +666,7 @@ test_channel_outbound_cell(void *arg) tor_free(p_cell); channel_free_all(); UNMOCK(scheduler_release_channel); + monotime_disable_test_mocking(); } /* Test inbound cell. The callstack is: diff --git a/src/test/test_channelpadding.c b/src/test/test_channelpadding.c index 90da2163a6..4261bc1b67 100644 --- a/src/test/test_channelpadding.c +++ b/src/test/test_channelpadding.c @@ -15,7 +15,6 @@ #include "channelpadding.h" #include "compat_libevent.h" #include "config.h" -#include <event2/event.h> #include "compat_time.h" #include "main.h" #include "networkstatus.h" @@ -65,7 +64,7 @@ mock_channel_write_cell_relay2(channel_t *chan, cell_t *cell) (void)chan; tried_to_write_cell++; channel_tls_handle_cell(cell, ((channel_tls_t*)relay1_relay2)->conn); - event_base_loopbreak(tor_libevent_get_base()); + tor_libevent_exit_loop_after_callback(tor_libevent_get_base()); return 0; } @@ -75,7 +74,7 @@ mock_channel_write_cell_relay1(channel_t *chan, cell_t *cell) (void)chan; tried_to_write_cell++; channel_tls_handle_cell(cell, ((channel_tls_t*)relay2_relay1)->conn); - event_base_loopbreak(tor_libevent_get_base()); + tor_libevent_exit_loop_after_callback(tor_libevent_get_base()); return 0; } @@ -85,7 +84,7 @@ mock_channel_write_cell_relay3(channel_t *chan, cell_t *cell) (void)chan; tried_to_write_cell++; channel_tls_handle_cell(cell, ((channel_tls_t*)client_relay3)->conn); - event_base_loopbreak(tor_libevent_get_base()); + tor_libevent_exit_loop_after_callback(tor_libevent_get_base()); return 0; } @@ -95,7 +94,7 @@ mock_channel_write_cell_client(channel_t *chan, cell_t *cell) (void)chan; tried_to_write_cell++; channel_tls_handle_cell(cell, ((channel_tls_t*)relay3_client)->conn); - event_base_loopbreak(tor_libevent_get_base()); + tor_libevent_exit_loop_after_callback(tor_libevent_get_base()); return 0; } @@ -105,7 +104,7 @@ mock_channel_write_cell(channel_t *chan, cell_t *cell) tried_to_write_cell++; channel_tls_handle_cell(cell, ((channel_tls_t*)chan)->conn); if (!dont_stop_libevent) - event_base_loopbreak(tor_libevent_get_base()); + tor_libevent_exit_loop_after_callback(tor_libevent_get_base()); return 0; } @@ -246,7 +245,7 @@ static void dummy_timer_cb(tor_timer_t *t, void *arg, const monotime_t *now_mono) { (void)t; (void)arg; (void)now_mono; - event_base_loopbreak(tor_libevent_get_base()); + tor_libevent_exit_loop_after_callback(tor_libevent_get_base()); return; } @@ -264,7 +263,8 @@ dummy_nop_timer(void) timer_schedule(dummy_timer, &timeout); - event_base_loop(tor_libevent_get_base(), 0); + tor_libevent_run_event_loop(tor_libevent_get_base(), 0); + timer_free(dummy_timer); } diff --git a/src/test/test_circuitlist.c b/src/test/test_circuitlist.c index d170009a9c..3794ffc2c6 100644 --- a/src/test/test_circuitlist.c +++ b/src/test/test_circuitlist.c @@ -9,6 +9,7 @@ #include "channel.h" #include "circuitbuild.h" #include "circuitlist.h" +#include "circuitmux_ewma.h" #include "hs_circuitmap.h" #include "test.h" #include "log_test_helpers.h" diff --git a/src/test/test_circuitmux.c b/src/test/test_circuitmux.c index 854f725054..c81d53ae51 100644 --- a/src/test/test_circuitmux.c +++ b/src/test/test_circuitmux.c @@ -3,14 +3,18 @@ #define TOR_CHANNEL_INTERNAL_ #define CIRCUITMUX_PRIVATE +#define CIRCUITMUX_EWMA_PRIVATE #define RELAY_PRIVATE #include "or.h" #include "channel.h" #include "circuitmux.h" +#include "circuitmux_ewma.h" #include "relay.h" #include "scheduler.h" #include "test.h" +#include <math.h> + /* XXXX duplicated function from test_circuitlist.c */ static channel_t * new_fake_channel(void) @@ -45,6 +49,7 @@ test_cmux_destroy_cell_queue(void *arg) cmux = circuitmux_alloc(); tt_assert(cmux); ch = new_fake_channel(); + circuitmux_set_policy(cmux, &ewma_policy); ch->has_queued_writes = has_queued_writes; ch->wide_circ_ids = 1; @@ -77,8 +82,50 @@ test_cmux_destroy_cell_queue(void *arg) tor_free(dc); } +static void +test_cmux_compute_ticks(void *arg) +{ + const int64_t NS_PER_S = 1000 * 1000 * 1000; + const int64_t START_NS = U64_LITERAL(1217709000)*NS_PER_S; + int64_t now; + double rem; + unsigned tick; + (void)arg; + circuitmux_ewma_free_all(); + monotime_enable_test_mocking(); + + monotime_coarse_set_mock_time_nsec(START_NS); + cell_ewma_initialize_ticks(); + const unsigned tick_zero = cell_ewma_get_current_tick_and_fraction(&rem); + tt_double_op(rem, OP_GT, -1e-9); + tt_double_op(rem, OP_LT, 1e-9); + + /* 1.5 second later and we should still be in the same tick. */ + now = START_NS + NS_PER_S + NS_PER_S/2; + monotime_coarse_set_mock_time_nsec(now); + tick = cell_ewma_get_current_tick_and_fraction(&rem); + tt_uint_op(tick, OP_EQ, tick_zero); +#ifdef USING_32BIT_MSEC_HACK + const double tolerance = .0005; +#else + const double tolerance = .00000001; +#endif + tt_double_op(fabs(rem - .15), OP_LT, tolerance); + + /* 25 second later and we should be in another tick. */ + now = START_NS + NS_PER_S * 25; + monotime_coarse_set_mock_time_nsec(now); + tick = cell_ewma_get_current_tick_and_fraction(&rem); + tt_uint_op(tick, OP_EQ, tick_zero + 2); + tt_double_op(fabs(rem - .5), OP_LT, tolerance); + + done: + ; +} + struct testcase_t circuitmux_tests[] = { { "destroy_cell_queue", test_cmux_destroy_cell_queue, TT_FORK, NULL, NULL }, + { "compute_ticks", test_cmux_compute_ticks, TT_FORK, NULL, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_compat_libevent.c b/src/test/test_compat_libevent.c index 7dd8e65194..85f69bd626 100644 --- a/src/test/test_compat_libevent.c +++ b/src/test/test_compat_libevent.c @@ -10,7 +10,6 @@ #include "compat_libevent.h" #include <event2/event.h> -#include <event2/thread.h> #include "log_test_helpers.h" @@ -122,10 +121,70 @@ test_compat_libevent_header_version(void *ignored) (void)0; } +/* Test for postloop events */ + +/* Event callback to increment a counter. */ +static void +increment_int_counter_cb(periodic_timer_t *timer, void *arg) +{ + (void)timer; + int *ctr = arg; + ++*ctr; +} + +static int activated_counter = 0; + +/* Mainloop event callback to activate another mainloop event */ +static void +activate_event_cb(mainloop_event_t *ev, void *arg) +{ + (void)ev; + mainloop_event_t **other_event = arg; + mainloop_event_activate(*other_event); + ++activated_counter; +} + +static void +test_compat_libevent_postloop_events(void *arg) +{ + (void)arg; + mainloop_event_t *a = NULL, *b = NULL; + periodic_timer_t *timed = NULL; + + tor_libevent_postfork(); + + /* If postloop events don't work, then these events will activate one + * another ad infinitum and, and the periodic event will never occur. */ + b = mainloop_event_postloop_new(activate_event_cb, &a); + a = mainloop_event_postloop_new(activate_event_cb, &b); + + int counter = 0; + struct timeval fifty_ms = { 0, 10 * 1000 }; + timed = periodic_timer_new(tor_libevent_get_base(), &fifty_ms, + increment_int_counter_cb, &counter); + + mainloop_event_activate(a); + int r; + do { + r = tor_libevent_run_event_loop(tor_libevent_get_base(), 0); + if (r == -1) + break; + } while (counter < 5); + + tt_int_op(activated_counter, OP_GE, 2); + + done: + mainloop_event_free(a); + mainloop_event_free(b); + periodic_timer_free(timed); +} + struct testcase_t compat_libevent_tests[] = { { "logging_callback", test_compat_libevent_logging_callback, TT_FORK, NULL, NULL }, { "header_version", test_compat_libevent_header_version, 0, NULL, NULL }, + { "postloop_events", test_compat_libevent_postloop_events, + TT_FORK, NULL, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_config.c b/src/test/test_config.c index 2bcc0cad7b..8662b832b8 100644 --- a/src/test/test_config.c +++ b/src/test/test_config.c @@ -20,12 +20,11 @@ #include "connection_edge.h" #include "test.h" #include "util.h" -#include "address.h" #include "connection_or.h" #include "control.h" #include "cpuworker.h" #include "dirserv.h" -#include "dirvote.h" +#include "dirauth/dirvote.h" #include "dns.h" #include "entrynodes.h" #include "transports.h" @@ -42,9 +41,6 @@ #include "routerlist.h" #include "routerset.h" #include "statefile.h" -#include "test.h" -#include "transports.h" -#include "util.h" #include "test_helpers.h" diff --git a/src/test/test_consdiffmgr.c b/src/test/test_consdiffmgr.c index a9a4b6a98e..3b91baca39 100644 --- a/src/test/test_consdiffmgr.c +++ b/src/test/test_consdiffmgr.c @@ -9,6 +9,7 @@ #include "consdiff.h" #include "consdiffmgr.h" #include "cpuworker.h" +#include "crypto_rand.h" #include "networkstatus.h" #include "routerparse.h" #include "workqueue.h" diff --git a/src/test/test_containers.c b/src/test/test_containers.c index c4dba73750..3fc3523af4 100644 --- a/src/test/test_containers.c +++ b/src/test/test_containers.c @@ -5,6 +5,7 @@ #include "orconfig.h" #include "or.h" +#include "crypto_rand.h" #include "fp_pair.h" #include "test.h" diff --git a/src/test/test_controller.c b/src/test/test_controller.c index 1c285bb3a2..1a350f66c0 100644 --- a/src/test/test_controller.c +++ b/src/test/test_controller.c @@ -1470,6 +1470,61 @@ test_download_status_bridge(void *arg) return; } +/** Set timeval to a mock date and time. This is necessary + * to make tor_gettimeofday() mockable. */ +static void +mock_tor_gettimeofday(struct timeval *timeval) +{ + timeval->tv_sec = 1523405073; + timeval->tv_usec = 271645; +} + +static void +test_current_time(void *arg) +{ + /* We just need one of these to pass, it doesn't matter what's in it */ + control_connection_t dummy; + /* Get results out */ + char *answer = NULL; + const char *errmsg = NULL; + + (void)arg; + + /* We need these for storing the (mock) time. */ + MOCK(tor_gettimeofday, mock_tor_gettimeofday); + struct timeval now; + tor_gettimeofday(&now); + char timebuf[ISO_TIME_LEN+1]; + + /* Case 1 - local time */ + format_local_iso_time_nospace(timebuf, (time_t)now.tv_sec); + getinfo_helper_current_time(&dummy, + "current-time/local", + &answer, &errmsg); + tt_ptr_op(answer, OP_NE, NULL); + tt_ptr_op(errmsg, OP_EQ, NULL); + tt_str_op(answer, OP_EQ, timebuf); + tor_free(answer); + errmsg = NULL; + + /* Case 2 - UTC time */ + format_iso_time_nospace(timebuf, (time_t)now.tv_sec); + getinfo_helper_current_time(&dummy, + "current-time/utc", + &answer, &errmsg); + tt_ptr_op(answer, OP_NE, NULL); + tt_ptr_op(errmsg, OP_EQ, NULL); + tt_str_op(answer, OP_EQ, timebuf); + tor_free(answer); + errmsg = NULL; + + done: + UNMOCK(tor_gettimeofday); + tor_free(answer); + + return; +} + struct testcase_t controller_tests[] = { { "add_onion_helper_keyarg_v2", test_add_onion_helper_keyarg_v2, 0, NULL, NULL }, @@ -1486,6 +1541,7 @@ struct testcase_t controller_tests[] = { NULL }, { "download_status_desc", test_download_status_desc, 0, NULL, NULL }, { "download_status_bridge", test_download_status_bridge, 0, NULL, NULL }, + { "current_time", test_current_time, 0, NULL, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_controller_events.c b/src/test/test_controller_events.c index 901ad7ab3d..e81aea8d66 100644 --- a/src/test/test_controller_events.c +++ b/src/test/test_controller_events.c @@ -12,79 +12,6 @@ #include "test.h" static void -help_test_bucket_note_empty(uint32_t expected_msec_since_midnight, - int tokens_before, size_t tokens_removed, - uint32_t msec_since_epoch) -{ - uint32_t timestamp_var = 0; - struct timeval tvnow; - tvnow.tv_sec = msec_since_epoch / 1000; - tvnow.tv_usec = (msec_since_epoch % 1000) * 1000; - connection_buckets_note_empty_ts(×tamp_var, tokens_before, - tokens_removed, &tvnow); - tt_int_op(expected_msec_since_midnight, OP_EQ, timestamp_var); - - done: - ; -} - -static void -test_cntev_bucket_note_empty(void *arg) -{ - (void)arg; - - /* Two cases with nothing to note, because bucket was empty before; - * 86442200 == 1970-01-02 00:00:42.200000 */ - help_test_bucket_note_empty(0, 0, 0, 86442200); - help_test_bucket_note_empty(0, -100, 100, 86442200); - - /* Nothing to note, because bucket has not been emptied. */ - help_test_bucket_note_empty(0, 101, 100, 86442200); - - /* Bucket was emptied, note 42200 msec since midnight. */ - help_test_bucket_note_empty(42200, 101, 101, 86442200); - help_test_bucket_note_empty(42200, 101, 102, 86442200); -} - -static void -test_cntev_bucket_millis_empty(void *arg) -{ - struct timeval tvnow; - (void)arg; - - /* 1970-01-02 00:00:42.200000 */ - tvnow.tv_sec = 86400 + 42; - tvnow.tv_usec = 200000; - - /* Bucket has not been refilled. */ - tt_int_op(0, OP_EQ, bucket_millis_empty(0, 42120, 0, 100, &tvnow)); - tt_int_op(0, OP_EQ, bucket_millis_empty(-10, 42120, -10, 100, &tvnow)); - - /* Bucket was not empty. */ - tt_int_op(0, OP_EQ, bucket_millis_empty(10, 42120, 20, 100, &tvnow)); - - /* Bucket has been emptied 80 msec ago and has just been refilled. */ - tt_int_op(80, OP_EQ, bucket_millis_empty(-20, 42120, -10, 100, &tvnow)); - tt_int_op(80, OP_EQ, bucket_millis_empty(-10, 42120, 0, 100, &tvnow)); - tt_int_op(80, OP_EQ, bucket_millis_empty(0, 42120, 10, 100, &tvnow)); - - /* Bucket has been emptied 180 msec ago, last refill was 100 msec ago - * which was insufficient to make it positive, so cap msec at 100. */ - tt_int_op(100, OP_EQ, bucket_millis_empty(0, 42020, 1, 100, &tvnow)); - - /* 1970-01-02 00:00:00:050000 */ - tvnow.tv_sec = 86400; - tvnow.tv_usec = 50000; - - /* Last emptied 30 msec before midnight, tvnow is 50 msec after - * midnight, that's 80 msec in total. */ - tt_int_op(80, OP_EQ, bucket_millis_empty(0, 86400000 - 30, 1, 100, &tvnow)); - - done: - ; -} - -static void add_testing_cell_stats_entry(circuit_t *circ, uint8_t command, unsigned int waiting_time, unsigned int removed, unsigned int exitward) @@ -395,8 +322,6 @@ test_cntev_event_mask(void *arg) { #name, test_cntev_ ## name, flags, 0, NULL } struct testcase_t controller_event_tests[] = { - TEST(bucket_note_empty, TT_FORK), - TEST(bucket_millis_empty, TT_FORK), TEST(sum_up_cell_stats, TT_FORK), TEST(append_cell_stats, TT_FORK), TEST(format_cell_stats, TT_FORK), diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c index 83d97f2867..bb2e340dd2 100644 --- a/src/test/test_crypto.c +++ b/src/test/test_crypto.c @@ -5,7 +5,7 @@ #include "orconfig.h" #define CRYPTO_CURVE25519_PRIVATE -#define CRYPTO_PRIVATE +#define CRYPTO_RAND_PRIVATE #include "or.h" #include "test.h" #include "aes.h" @@ -13,6 +13,7 @@ #include "siphash.h" #include "crypto_curve25519.h" #include "crypto_ed25519.h" +#include "crypto_rand.h" #include "ed25519_vectors.inc" /** Run unit tests for Diffie-Hellman functionality. */ diff --git a/src/test/test_crypto_openssl.c b/src/test/test_crypto_openssl.c index 090cb4242b..a016277508 100644 --- a/src/test/test_crypto_openssl.c +++ b/src/test/test_crypto_openssl.c @@ -5,9 +5,9 @@ #include "orconfig.h" -#define CRYPTO_PRIVATE +#define CRYPTO_RAND_PRIVATE -#include "crypto.h" +#include "crypto_rand.h" #include "util.h" #include "util_format.h" #include "compat.h" diff --git a/src/test/test_crypto_slow.c b/src/test/test_crypto_slow.c index 2afb71ff5a..0e1f5bd227 100644 --- a/src/test/test_crypto_slow.c +++ b/src/test/test_crypto_slow.c @@ -9,6 +9,7 @@ #include "test.h" #include "crypto_s2k.h" #include "crypto_pwbox.h" +#include "crypto_rand.h" #if defined(HAVE_LIBSCRYPT_H) && defined(HAVE_LIBSCRYPT_SCRYPT) #define HAVE_LIBSCRYPT diff --git a/src/test/test_dir.c b/src/test/test_dir.c index a33b23bcc9..37b015b72d 100644 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@ -23,9 +23,10 @@ #include "config.h" #include "control.h" #include "crypto_ed25519.h" +#include "crypto_rand.h" #include "directory.h" #include "dirserv.h" -#include "dirvote.h" +#include "dirauth/dirvote.h" #include "entrynodes.h" #include "hibernate.h" #include "memarea.h" @@ -35,12 +36,13 @@ #include "routerlist.h" #include "routerparse.h" #include "routerset.h" -#include "shared_random_state.h" +#include "dirauth/shared_random_state.h" #include "test.h" #include "test_dir_common.h" #include "torcert.h" #include "relay.h" #include "log_test_helpers.h" +#include "voting_schedule.h" #define NS_MODULE dir @@ -1499,6 +1501,13 @@ test_dir_measured_bw_kb(void *arg) "bw=1024 junk=007\n", "misc=junk node_id=$557365204145532d32353620696e73746561642e " "bw=1024 junk=007\n", + /* check whether node_id can be at the end */ + "bw=1024 node_id=$557365204145532d32353620696e73746561642e\n", + /* check whether node_id can be at the end and bw has something in front*/ + "foo=bar bw=1024 node_id=$557365204145532d32353620696e73746561642e\n", + /* check whether node_id can be at the end and something in the + * in the middle of bw and node_id */ + "bw=1024 foo=bar node_id=$557365204145532d32353620696e73746561642e\n", "end" }; const char *lines_fail[] = { @@ -1538,12 +1547,18 @@ test_dir_measured_bw_kb(void *arg) (void)arg; for (i = 0; strcmp(lines_fail[i], "end"); i++) { //fprintf(stderr, "Testing: %s\n", lines_fail[i]); - tt_int_op(measured_bw_line_parse(&mbwl, lines_fail[i]), OP_EQ, -1); + /* Testing only with line_is_after_headers = 1. Tests with + * line_is_after_headers = 0 in + * test_dir_measured_bw_kb_line_is_after_headers */ + tt_assert(measured_bw_line_parse(&mbwl, lines_fail[i], 1) == -1); } for (i = 0; strcmp(lines_pass[i], "end"); i++) { //fprintf(stderr, "Testing: %s %d\n", lines_pass[i], TOR_ISSPACE('\n')); - tt_int_op(measured_bw_line_parse(&mbwl, lines_pass[i]), OP_EQ, 0); + /* Testing only with line_is_after_headers = 1. Tests with + * line_is_after_headers = 0 in + * test_dir_measured_bw_kb_line_is_after_headers */ + tt_assert(measured_bw_line_parse(&mbwl, lines_pass[i], 1) == 0); tt_assert(mbwl.bw_kb == 1024); tt_assert(strcmp(mbwl.node_hex, "557365204145532d32353620696e73746561642e") == 0); @@ -1555,7 +1570,7 @@ test_dir_measured_bw_kb(void *arg) /* Test dirserv_read_measured_bandwidths */ static void -test_dir_dirserv_read_measured_bandwidths(void *arg) +test_dir_dirserv_read_measured_bandwidths_empty(void *arg) { char *fname=NULL; (void)arg; @@ -1572,6 +1587,129 @@ test_dir_dirserv_read_measured_bandwidths(void *arg) teardown_capture_of_logs(); } +/* Unit tests for measured_bw_line_parse using line_is_after_headers flag. + * When the end of the header is detected (a first complete bw line is parsed), + * incomplete lines fail and give warnings, but do not give warnings if + * the header is not ended, allowing to ignore additional header lines. */ +static void +test_dir_measured_bw_kb_line_is_after_headers(void *arg) +{ + (void)arg; + measured_bw_line_t mbwl; + const char *line_pass = \ + "node_id=$557365204145532d32353620696e73746561642e bw=1024\n"; + int i; + const char *lines_fail[] = { + "node_id=$557365204145532d32353620696e73746561642e \n", + "bw=1024\n", + "rtt=300\n", + "end" + }; + + setup_capture_of_logs(LOG_DEBUG); + + /* Test bw lines when header has ended */ + for (i = 0; strcmp(lines_fail[i], "end"); i++) { + tt_assert(measured_bw_line_parse(&mbwl, lines_fail[i], 1) == -1); + expect_log_msg_containing("Incomplete line in bandwidth file:"); + mock_clean_saved_logs(); + } + + tt_assert(measured_bw_line_parse(&mbwl, line_pass, 1) == 0); + + /* Test bw lines when header has not ended */ + for (i = 0; strcmp(lines_fail[i], "end"); i++) { + tt_assert(measured_bw_line_parse(&mbwl, lines_fail[i], 0) == -1); + expect_log_msg_containing("Missing bw or node_id in bandwidth file line:"); + mock_clean_saved_logs(); + } + + tt_assert(measured_bw_line_parse(&mbwl, line_pass, 0) == 0); + + done: + teardown_capture_of_logs(); +} + +/* Test dirserv_read_measured_bandwidths with whole files. */ +static void +test_dir_dirserv_read_measured_bandwidths(void *arg) +{ + (void)arg; + char *content = NULL; + time_t timestamp = time(NULL); + char *fname = tor_strdup(get_fname("V3BandwidthsFile")); + + /* Test Torflow file only with timestamp*/ + tor_asprintf(&content, "%ld", (long)timestamp); + write_str_to_file(fname, content, 0); + tor_free(content); + tt_int_op(-1, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL)); + + /* Test Torflow file with timestamp followed by '\n' */ + tor_asprintf(&content, "%ld\n", (long)timestamp); + write_str_to_file(fname, content, 0); + tor_free(content); + tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL)); + + /* Test Torflow complete file*/ + const char *torflow_relay_lines= + "node_id=$557365204145532d32353620696e73746561642e bw=1024 " + "nick=Test measured_at=1523911725 updated_at=1523911725 " + "pid_error=4.11374090719 pid_error_sum=4.11374090719 " + "pid_bw=57136645 pid_delta=2.12168374577 circ_fail=0.2 " + "scanner=/filepath\n"; + + tor_asprintf(&content, "%ld\n%s", (long)timestamp, torflow_relay_lines); + write_str_to_file(fname, content, 0); + tor_free(content); + tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL)); + + /* Test Torflow complete file including v1.1.0 headers */ + const char *v110_header_lines= + "version=1.1.0\n" + "software=sbws\n" + "software_version=0.1.0\n" + "generator_started=2018-05-08T16:13:25\n" + "earliest_bandwidth=2018-05-08T16:13:26\n" + "====\n"; + + tor_asprintf(&content, "%ld\n%s%s", (long)timestamp, v110_header_lines, + torflow_relay_lines); + write_str_to_file(fname, content, 0); + tor_free(content); + tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL)); + + /* Test Torflow with additional headers afer a correct bw line */ + tor_asprintf(&content, "%ld\n%s%s", (long)timestamp, torflow_relay_lines, + v110_header_lines); + write_str_to_file(fname, content, 0); + tor_free(content); + tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL)); + + /* Test Torflow with additional headers afer a correct bw line and more + * bw lines after the headers. */ + tor_asprintf(&content, "%ld\n%s%s%s", (long)timestamp, torflow_relay_lines, + v110_header_lines, torflow_relay_lines); + write_str_to_file(fname, content, 0); + tor_free(content); + tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL)); + + /* Test sbws file */ + const char *sbws_relay_lines= + "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 " + "master_key_ed25519=YaqV4vbvPYKucElk297eVdNArDz9HtIwUoIeo0+cVIpQ " + "bw=760 nick=Test rtt=380 time=2018-05-08T16:13:26\n"; + + tor_asprintf(&content, "%ld\n%s%s", (long)timestamp, v110_header_lines, + sbws_relay_lines); + write_str_to_file(fname, content, 0); + tor_free(content); + tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL)); + + done: + tor_free(fname); +} + #define MBWC_INIT_TIME 1000 /** Do the measured bandwidth cache unit test */ @@ -2398,7 +2536,7 @@ test_a_networkstatus( sign_skey_2 = crypto_pk_new(); sign_skey_3 = crypto_pk_new(); sign_skey_leg1 = pk_generate(4); - dirvote_recalculate_timing(get_options(), now); + voting_schedule_recalculate_timing(get_options(), now); sr_state_init(0, 0); tt_assert(!crypto_pk_read_private_key_from_string(sign_skey_1, @@ -2936,8 +3074,9 @@ gen_routerstatus_for_umbw(int idx, time_t now) rs->addr = 0x99008801; rs->or_port = 443; rs->dir_port = 8000; - /* all flags but running cleared */ + /* all flags but running and valid cleared */ rs->is_flagged_running = 1; + rs->is_valid = 1; /* * This one has measured bandwidth below the clip cutoff, and * so shouldn't be clipped; we'll have to test that it isn't @@ -3010,8 +3149,9 @@ gen_routerstatus_for_umbw(int idx, time_t now) rs->addr = 0xC0000203; rs->or_port = 500; rs->dir_port = 1999; - /* all flags but running cleared */ + /* all flags but running and valid cleared */ rs->is_flagged_running = 1; + rs->is_valid = 1; /* * This one has unmeasured bandwidth below the clip cutoff, and * so shouldn't be clipped; we'll have to test that it isn't @@ -3033,7 +3173,7 @@ gen_routerstatus_for_umbw(int idx, time_t now) if (vrs) { vrs->microdesc = tor_malloc_zero(sizeof(vote_microdesc_hash_t)); tor_asprintf(&vrs->microdesc->microdesc_hash_line, - "m 9,10,11,12,13,14,15,16,17 " + "m 25,26,27,28 " "sha256=xyzajkldsdsajdadlsdjaslsdksdjlsdjsdaskdaaa%d\n", idx); } @@ -3059,7 +3199,7 @@ vote_tweaks_for_umbw(networkstatus_t *v, int voter, time_t now) smartlist_clear(v->supported_methods); /* Method 17 is MIN_METHOD_TO_CLIP_UNMEASURED_BW_KB */ smartlist_split_string(v->supported_methods, - "1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17", + "25 26 27 28", NULL, 0, -1); /* If we're using a non-default clip bandwidth, add it to net_params */ if (alternate_clip_bw > 0) { @@ -3221,9 +3361,9 @@ test_routerstatus_for_umbw(routerstatus_t *rs, time_t now) tt_assert(!rs->is_fast); tt_assert(!rs->is_possible_guard); tt_assert(!rs->is_stable); - /* (If it wasn't running it wouldn't be here) */ + /* (If it wasn't running and valid it wouldn't be here) */ tt_assert(rs->is_flagged_running); - tt_assert(!rs->is_valid); + tt_assert(rs->is_valid); tt_assert(!rs->is_named); /* This one should have measured bandwidth below the clip cutoff */ tt_assert(rs->has_bandwidth); @@ -4082,34 +4222,19 @@ test_dir_download_status_increment(void *arg) DL_WANT_ANY_DIRSERVER, DL_SCHED_INCREMENT_ATTEMPT, 0, 0 }; - int no_delay = 0; - int delay0 = -1; - int delay1 = -1; - int delay2 = -1; - smartlist_t *schedule = smartlist_new(); - smartlist_t *schedule_no_initial_delay = smartlist_new(); or_options_t test_options; time_t current_time = time(NULL); - /* Provide some values for the schedules */ - delay0 = 10; - delay1 = 99; - delay2 = 20; - - /* Make the schedules */ - smartlist_add(schedule, (void *)&delay0); - smartlist_add(schedule, (void *)&delay1); - smartlist_add(schedule, (void *)&delay2); - - smartlist_add(schedule_no_initial_delay, (void *)&no_delay); - smartlist_add(schedule_no_initial_delay, (void *)&delay1); - smartlist_add(schedule_no_initial_delay, (void *)&delay2); + const int delay0 = 10; + const int no_delay = 0; + const int schedule = 10; + const int schedule_no_initial_delay = 0; /* Put it in the options */ mock_options = &test_options; reset_options(mock_options, &mock_get_options_calls); - mock_options->TestingBridgeBootstrapDownloadSchedule = schedule; - mock_options->TestingClientDownloadSchedule = schedule; + mock_options->TestingBridgeBootstrapDownloadInitialDelay = schedule; + mock_options->TestingClientDownloadInitialDelay = schedule; MOCK(get_options, mock_get_options); @@ -4117,13 +4242,13 @@ test_dir_download_status_increment(void *arg) * whether or not it was reset before being used */ /* regression test for 17750: no initial delay */ - mock_options->TestingClientDownloadSchedule = schedule_no_initial_delay; + mock_options->TestingClientDownloadInitialDelay = schedule_no_initial_delay; mock_get_options_calls = 0; /* we really want to test that it's equal to time(NULL) + delay0, but that's * an unrealiable test, because time(NULL) might change. */ /* regression test for 17750: exponential, no initial delay */ - mock_options->TestingClientDownloadSchedule = schedule_no_initial_delay; + mock_options->TestingClientDownloadInitialDelay = schedule_no_initial_delay; mock_get_options_calls = 0; /* we really want to test that it's equal to time(NULL) + delay0, but that's * an unrealiable test, because time(NULL) might change. */ @@ -4136,7 +4261,7 @@ test_dir_download_status_increment(void *arg) tt_int_op(mock_get_options_calls, OP_GE, 1); /* regression test for 17750: exponential, initial delay */ - mock_options->TestingClientDownloadSchedule = schedule; + mock_options->TestingClientDownloadInitialDelay = schedule; mock_get_options_calls = 0; /* we really want to test that it's equal to time(NULL) + delay0, but that's * an unrealiable test, because time(NULL) might change. */ @@ -4149,9 +4274,6 @@ test_dir_download_status_increment(void *arg) tt_int_op(mock_get_options_calls, OP_GE, 1); done: - /* the pointers in schedule are allocated on the stack */ - smartlist_free(schedule); - smartlist_free(schedule_no_initial_delay); UNMOCK(get_options); mock_options = NULL; mock_get_options_calls = 0; @@ -5469,7 +5591,7 @@ mock_num_bridges_usable(int use_maybe_reachable) * fallbacks. */ static void -test_dir_find_dl_schedule(void* data) +test_dir_find_dl_min_delay(void* data) { const char *str = (const char *)data; @@ -5502,44 +5624,45 @@ test_dir_find_dl_schedule(void* data) mock_num_bridges_usable); download_status_t dls; - smartlist_t server, client, server_cons, client_cons; - smartlist_t client_boot_auth_only_cons, client_boot_auth_cons; - smartlist_t client_boot_fallback_cons, bridge, bridge_bootstrap; + + const int server=10, client=20, server_cons=30, client_cons=40; + const int client_boot_auth_only_cons=50, client_boot_auth_cons=60; + const int client_boot_fallback_cons=70, bridge=80, bridge_bootstrap=90; mock_options = tor_malloc(sizeof(or_options_t)); reset_options(mock_options, &mock_get_options_calls); MOCK(get_options, mock_get_options); - mock_options->TestingServerDownloadSchedule = &server; - mock_options->TestingClientDownloadSchedule = &client; - mock_options->TestingServerConsensusDownloadSchedule = &server_cons; - mock_options->TestingClientConsensusDownloadSchedule = &client_cons; - mock_options->ClientBootstrapConsensusAuthorityOnlyDownloadSchedule = - &client_boot_auth_only_cons; - mock_options->ClientBootstrapConsensusAuthorityDownloadSchedule = - &client_boot_auth_cons; - mock_options->ClientBootstrapConsensusFallbackDownloadSchedule = - &client_boot_fallback_cons; - mock_options->TestingBridgeDownloadSchedule = &bridge; - mock_options->TestingBridgeBootstrapDownloadSchedule = &bridge_bootstrap; + mock_options->TestingServerDownloadInitialDelay = server; + mock_options->TestingClientDownloadInitialDelay = client; + mock_options->TestingServerConsensusDownloadInitialDelay = server_cons; + mock_options->TestingClientConsensusDownloadInitialDelay = client_cons; + mock_options->ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay = + client_boot_auth_only_cons; + mock_options->ClientBootstrapConsensusAuthorityDownloadInitialDelay = + client_boot_auth_cons; + mock_options->ClientBootstrapConsensusFallbackDownloadInitialDelay = + client_boot_fallback_cons; + mock_options->TestingBridgeDownloadInitialDelay = bridge; + mock_options->TestingBridgeBootstrapDownloadInitialDelay = bridge_bootstrap; dls.schedule = DL_SCHED_GENERIC; /* client */ mock_options->ClientOnly = 1; - tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, &client); + tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, client); mock_options->ClientOnly = 0; /* dir mode */ mock_options->DirPort_set = 1; mock_options->DirCache = 1; - tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, &server); + tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, server); mock_options->DirPort_set = 0; mock_options->DirCache = 0; dls.schedule = DL_SCHED_CONSENSUS; /* public server mode */ mock_options->ORPort_set = 1; - tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, &server_cons); + tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, server_cons); mock_options->ORPort_set = 0; /* client and bridge modes */ @@ -5548,30 +5671,30 @@ test_dir_find_dl_schedule(void* data) dls.want_authority = 1; /* client */ mock_options->ClientOnly = 1; - tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, - &client_boot_auth_cons); + tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, + client_boot_auth_cons); mock_options->ClientOnly = 0; /* bridge relay */ mock_options->ORPort_set = 1; mock_options->BridgeRelay = 1; - tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, - &client_boot_auth_cons); + tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, + client_boot_auth_cons); mock_options->ORPort_set = 0; mock_options->BridgeRelay = 0; dls.want_authority = 0; /* client */ mock_options->ClientOnly = 1; - tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, - &client_boot_fallback_cons); + tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, + client_boot_fallback_cons); mock_options->ClientOnly = 0; /* bridge relay */ mock_options->ORPort_set = 1; mock_options->BridgeRelay = 1; - tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, - &client_boot_fallback_cons); + tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, + client_boot_fallback_cons); mock_options->ORPort_set = 0; mock_options->BridgeRelay = 0; @@ -5579,30 +5702,30 @@ test_dir_find_dl_schedule(void* data) /* dls.want_authority is ignored */ /* client */ mock_options->ClientOnly = 1; - tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, - &client_boot_auth_only_cons); + tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, + client_boot_auth_only_cons); mock_options->ClientOnly = 0; /* bridge relay */ mock_options->ORPort_set = 1; mock_options->BridgeRelay = 1; - tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, - &client_boot_auth_only_cons); + tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, + client_boot_auth_only_cons); mock_options->ORPort_set = 0; mock_options->BridgeRelay = 0; } } else { /* client */ mock_options->ClientOnly = 1; - tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, - &client_cons); + tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, + client_cons); mock_options->ClientOnly = 0; /* bridge relay */ mock_options->ORPort_set = 1; mock_options->BridgeRelay = 1; - tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, - &client_cons); + tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, + client_cons); mock_options->ORPort_set = 0; mock_options->BridgeRelay = 0; } @@ -5612,9 +5735,9 @@ test_dir_find_dl_schedule(void* data) mock_options->ClientOnly = 1; mock_options->UseBridges = 1; if (num_bridges_usable(0) > 0) { - tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, &bridge); + tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, bridge); } else { - tt_ptr_op(find_dl_schedule(&dls, mock_options), OP_EQ, &bridge_bootstrap); + tt_int_op(find_dl_min_delay(&dls, mock_options), OP_EQ, bridge_bootstrap); } done: @@ -5634,9 +5757,8 @@ test_dir_assumed_flags(void *arg) memarea_t *area = memarea_new(); routerstatus_t *rs = NULL; - /* First, we should always assume that the Running flag is set, even - * when it isn't listed, since the consensus method is always - * higher than 4. */ + /* We can assume that consensus method is higher than 24, so Running and + * Valid are always implicitly set */ const char *str1 = "r example hereiswhereyouridentitygoes 2015-08-30 12:00:00 " "192.168.0.1 9001 0\n" @@ -5645,17 +5767,6 @@ test_dir_assumed_flags(void *arg) const char *cp = str1; rs = routerstatus_parse_entry_from_string(area, &cp, tokens, NULL, NULL, - 23, FLAV_MICRODESC); - tt_assert(rs); - tt_assert(rs->is_flagged_running); - tt_assert(! rs->is_valid); - tt_assert(! rs->is_exit); - tt_assert(rs->is_fast); - routerstatus_free(rs); - - /* With method 24 or later, we can assume "valid" is set. */ - cp = str1; - rs = routerstatus_parse_entry_from_string(area, &cp, tokens, NULL, NULL, 24, FLAV_MICRODESC); tt_assert(rs); tt_assert(rs->is_flagged_running); @@ -5788,22 +5899,10 @@ test_dir_networkstatus_consensus_has_ipv6(void *arg) /* Test the bounds for A lines in the NS consensus */ mock_options->UseMicrodescriptors = 0; - mock_networkstatus->consensus_method = MIN_METHOD_FOR_A_LINES; - has_ipv6 = networkstatus_consensus_has_ipv6(get_options()); - tt_assert(has_ipv6); - - mock_networkstatus->consensus_method = MIN_METHOD_FOR_A_LINES + 1; + mock_networkstatus->consensus_method = MIN_SUPPORTED_CONSENSUS_METHOD; has_ipv6 = networkstatus_consensus_has_ipv6(get_options()); tt_assert(has_ipv6); - mock_networkstatus->consensus_method = MIN_METHOD_FOR_A_LINES + 20; - has_ipv6 = networkstatus_consensus_has_ipv6(get_options()); - tt_assert(has_ipv6); - - mock_networkstatus->consensus_method = MIN_METHOD_FOR_A_LINES - 1; - has_ipv6 = networkstatus_consensus_has_ipv6(get_options()); - tt_assert(!has_ipv6); - /* Test the bounds for A lines in the microdesc consensus */ mock_options->UseMicrodescriptors = 1; @@ -5812,6 +5911,10 @@ test_dir_networkstatus_consensus_has_ipv6(void *arg) has_ipv6 = networkstatus_consensus_has_ipv6(get_options()); tt_assert(has_ipv6); + mock_networkstatus->consensus_method = MAX_SUPPORTED_CONSENSUS_METHOD + 20; + has_ipv6 = networkstatus_consensus_has_ipv6(get_options()); + tt_assert(has_ipv6); + mock_networkstatus->consensus_method = MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS + 1; has_ipv6 = networkstatus_consensus_has_ipv6(get_options()); @@ -5926,9 +6029,11 @@ struct testcase_t dir_tests[] = { DIR_LEGACY(versions), DIR_LEGACY(fp_pairs), DIR(split_fps, 0), - DIR_LEGACY(dirserv_read_measured_bandwidths), + DIR_LEGACY(dirserv_read_measured_bandwidths_empty), DIR_LEGACY(measured_bw_kb), + DIR_LEGACY(measured_bw_kb_line_is_after_headers), DIR_LEGACY(measured_bw_kb_cache), + DIR_LEGACY(dirserv_read_measured_bandwidths), DIR_LEGACY(param_voting), DIR(param_voting_lookup, 0), DIR_LEGACY(v3_networkstatus), @@ -5960,14 +6065,14 @@ struct testcase_t dir_tests[] = { DIR(dump_unparseable_descriptors, 0), DIR(populate_dump_desc_fifo, 0), DIR(populate_dump_desc_fifo_2, 0), - DIR_ARG(find_dl_schedule, TT_FORK, "bfd"), - DIR_ARG(find_dl_schedule, TT_FORK, "bad"), - DIR_ARG(find_dl_schedule, TT_FORK, "cfd"), - DIR_ARG(find_dl_schedule, TT_FORK, "cad"), - DIR_ARG(find_dl_schedule, TT_FORK, "bfr"), - DIR_ARG(find_dl_schedule, TT_FORK, "bar"), - DIR_ARG(find_dl_schedule, TT_FORK, "cfr"), - DIR_ARG(find_dl_schedule, TT_FORK, "car"), + DIR_ARG(find_dl_min_delay, TT_FORK, "bfd"), + DIR_ARG(find_dl_min_delay, TT_FORK, "bad"), + DIR_ARG(find_dl_min_delay, TT_FORK, "cfd"), + DIR_ARG(find_dl_min_delay, TT_FORK, "cad"), + DIR_ARG(find_dl_min_delay, TT_FORK, "bfr"), + DIR_ARG(find_dl_min_delay, TT_FORK, "bar"), + DIR_ARG(find_dl_min_delay, TT_FORK, "cfr"), + DIR_ARG(find_dl_min_delay, TT_FORK, "car"), DIR(assumed_flags, 0), DIR(networkstatus_compute_bw_weights_v10, 0), DIR(platform_str, 0), diff --git a/src/test/test_dir_common.c b/src/test/test_dir_common.c index fdf43533a8..230410f7fa 100644 --- a/src/test/test_dir_common.c +++ b/src/test/test_dir_common.c @@ -5,14 +5,14 @@ #include "orconfig.h" #define DIRVOTE_PRIVATE -#include "crypto.h" #include "test.h" #include "container.h" #include "or.h" -#include "dirvote.h" +#include "dirauth/dirvote.h" #include "nodelist.h" #include "routerlist.h" #include "test_dir_common.h" +#include "voting_schedule.h" void dir_common_setup_vote(networkstatus_t **vote, time_t now); networkstatus_t * dir_common_add_rs_and_parse(networkstatus_t *vote, diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c index ca64dce5fe..688d26bdc1 100644 --- a/src/test/test_dir_handle_get.c +++ b/src/test/test_dir_handle_get.c @@ -16,7 +16,6 @@ #include "directory.h" #include "test.h" #include "compress.h" -#include "connection.h" #include "rendcommon.h" #include "rendcache.h" #include "router.h" @@ -31,8 +30,9 @@ #include "proto_http.h" #include "geoip.h" #include "dirserv.h" -#include "dirvote.h" +#include "dirauth/dirvote.h" #include "log_test_helpers.h" +#include "voting_schedule.h" #ifdef _WIN32 /* For mkdir() */ @@ -2057,7 +2057,7 @@ test_dir_handle_get_status_vote_d(void* data) mock_options->TestingV3AuthInitialDistDelay = 1; time_t now = 1441223455 -1; - dirvote_recalculate_timing(mock_options, now); + voting_schedule_recalculate_timing(mock_options, now); const char *msg_out = NULL; int status_out = 0; @@ -2403,7 +2403,7 @@ test_dir_handle_get_status_vote_next_authority(void* data) mock_options->TestingV3AuthInitialDistDelay = 1; time_t now = 1441223455 -1; - dirvote_recalculate_timing(mock_options, now); + voting_schedule_recalculate_timing(mock_options, now); struct pending_vote_t *vote = dirvote_add_vote(VOTE_BODY_V3, &msg_out, &status_out); @@ -2482,7 +2482,7 @@ test_dir_handle_get_status_vote_current_authority(void* data) mock_options->TestingV3AuthInitialDistDelay = 1; time_t now = 1441223455; - dirvote_recalculate_timing(mock_options, now-1); + voting_schedule_recalculate_timing(mock_options, now-1); struct pending_vote_t *vote = dirvote_add_vote(VOTE_BODY_V3, &msg_out, &status_out); diff --git a/src/test/test_dos.c b/src/test/test_dos.c index cb9d9e559c..8ae967f3ae 100644 --- a/src/test/test_dos.c +++ b/src/test/test_dos.c @@ -8,6 +8,7 @@ #include "or.h" #include "dos.h" #include "circuitlist.h" +#include "crypto_rand.h" #include "geoip.h" #include "channel.h" #include "microdesc.h" diff --git a/src/test/test_entrynodes.c b/src/test/test_entrynodes.c index 505e09e36f..d8bd8e328b 100644 --- a/src/test/test_entrynodes.c +++ b/src/test/test_entrynodes.c @@ -4,6 +4,7 @@ #include "orconfig.h" #define CIRCUITLIST_PRIVATE +#define CIRCUITBUILD_PRIVATE #define STATEFILE_PRIVATE #define ENTRYNODES_PRIVATE #define ROUTERLIST_PRIVATE @@ -14,8 +15,10 @@ #include "bridges.h" #include "circuitlist.h" +#include "circuitbuild.h" #include "config.h" #include "confparse.h" +#include "crypto_rand.h" #include "directory.h" #include "entrynodes.h" #include "nodelist.h" @@ -74,6 +77,17 @@ bfn_mock_node_get_by_id(const char *id) return NULL; } +/* Helper function to free a test node. */ +static void +test_node_free(node_t *n) +{ + tor_free(n->rs); + tor_free(n->md->onion_curve25519_pkey); + short_policy_free(n->md->exit_policy); + tor_free(n->md); + tor_free(n); +} + /* Unittest cleanup function: Cleanup the fake network. */ static int big_fake_network_cleanup(const struct testcase_t *testcase, void *ptr) @@ -83,9 +97,7 @@ big_fake_network_cleanup(const struct testcase_t *testcase, void *ptr) if (big_fake_net_nodes) { SMARTLIST_FOREACH(big_fake_net_nodes, node_t *, n, { - tor_free(n->rs); - tor_free(n->md); - tor_free(n); + test_node_free(n); }); smartlist_free(big_fake_net_nodes); } @@ -113,9 +125,18 @@ big_fake_network_setup(const struct testcase_t *testcase) big_fake_net_nodes = smartlist_new(); for (i = 0; i < N_NODES; ++i) { + curve25519_secret_key_t curve25519_secret_key; + node_t *n = tor_malloc_zero(sizeof(node_t)); n->md = tor_malloc_zero(sizeof(microdesc_t)); + /* Generate curve25519 key for this node */ + n->md->onion_curve25519_pkey = + tor_malloc_zero(sizeof(curve25519_public_key_t)); + curve25519_secret_key_generate(&curve25519_secret_key, 0); + curve25519_public_key_generate(n->md->onion_curve25519_pkey, + &curve25519_secret_key); + crypto_rand(n->identity, sizeof(n->identity)); n->rs = tor_malloc_zero(sizeof(routerstatus_t)); @@ -135,8 +156,8 @@ big_fake_network_setup(const struct testcase_t *testcase) { char nickname_binary[8]; crypto_rand(nickname_binary, sizeof(nickname_binary)); - base64_encode(n->rs->nickname, sizeof(n->rs->nickname), - nickname_binary, sizeof(nickname_binary), 0); + base32_encode(n->rs->nickname, sizeof(n->rs->nickname), + nickname_binary, sizeof(nickname_binary)); } /* Call half of the nodes a possible guard. */ @@ -144,6 +165,12 @@ big_fake_network_setup(const struct testcase_t *testcase) n->is_possible_guard = 1; n->rs->guardfraction_percentage = 100; n->rs->has_guardfraction = 1; + n->rs->is_possible_guard = 1; + } + + /* Make some of these nodes a possible exit */ + if (i % 7 == 0) { + n->md->exit_policy = parse_short_policy("accept 443"); } smartlist_add(big_fake_net_nodes, n); @@ -1075,9 +1102,7 @@ test_entry_guard_expand_sample_small_net(void *arg) /* Fun corner case: not enough guards to make up our whole sample size. */ SMARTLIST_FOREACH(big_fake_net_nodes, node_t *, n, { if (n_sl_idx >= 15) { - tor_free(n->rs); - tor_free(n->md); - tor_free(n); + test_node_free(n); SMARTLIST_DEL_CURRENT(big_fake_net_nodes, n); } else { n->rs->addr = 0; // make the filter reject this. @@ -1174,9 +1199,7 @@ test_entry_guard_update_from_consensus_status(void *arg) node_t *n = (node_t*) bfn_mock_node_get_by_id(g->identity); tt_assert(n); smartlist_remove(big_fake_net_nodes, n); - tor_free(n->rs); - tor_free(n->md); - tor_free(n); + test_node_free(n); } update_approx_time(start + 300); sampled_guards_update_from_consensus(gs); @@ -2685,6 +2708,23 @@ test_enty_guard_should_expire_waiting(void *arg) tor_free(fake_state); } +/** Test that the number of primary guards can be controlled using torrc */ +static void +test_entry_guard_number_of_primaries(void *arg) +{ + (void) arg; + + /* Get default value */ + tt_int_op(get_n_primary_guards(), OP_EQ, DFLT_N_PRIMARY_GUARDS); + + /* Set number of primaries using torrc */ + get_options_mutable()->NumPrimaryGuards = 42; + tt_int_op(get_n_primary_guards(), OP_EQ, 42); + + done: + ; +} + static void mock_directory_initiate_request(directory_request_t *req) { @@ -2793,6 +2833,161 @@ test_entry_guard_outdated_dirserver_exclusion(void *arg) } } +/** Test helper to extend the <b>oc</b> circuit path <b>n</b> times and then + * ensure that the circuit is now complete. */ +static void +helper_extend_circuit_path_n_times(origin_circuit_t *oc, int n) +{ + int retval; + int i; + + /* Extend path n times */ + for (i = 0 ; i < n ; i++) { + retval = onion_extend_cpath(oc); + tt_int_op(retval, OP_EQ, 0); + tt_int_op(circuit_get_cpath_len(oc), OP_EQ, i+1); + } + + /* Now do it one last time and see that circ is complete */ + retval = onion_extend_cpath(oc); + tt_int_op(retval, OP_EQ, 1); + + done: + ; +} + +/** Test for basic Tor path selection. Makes sure we build 3-hop circuits. */ +static void +test_entry_guard_basic_path_selection(void *arg) +{ + (void) arg; + + int retval; + + /* Enable entry guards */ + or_options_t *options = get_options_mutable(); + options->UseEntryGuards = 1; + + /* disables /16 check since all nodes have the same addr... */ + options->EnforceDistinctSubnets = 0; + + /* Create our circuit */ + circuit_t *circ = dummy_origin_circuit_new(30); + origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ); + oc->build_state = tor_malloc_zero(sizeof(cpath_build_state_t)); + + /* First pick the exit and pin it on the build_state */ + retval = onion_pick_cpath_exit(oc, NULL, 0); + tt_int_op(retval, OP_EQ, 0); + + /* Extend path 3 times. First we pick guard, then middle, then exit. */ + helper_extend_circuit_path_n_times(oc, 3); + + done: + circuit_free_(circ); +} + +/** Test helper to build an L2 and L3 vanguard list. The vanguard lists + * produced should be completely disjoint. */ +static void +helper_setup_vanguard_list(or_options_t *options) +{ + int i = 0; + + /* Add some nodes to the vanguard L2 list */ + options->HSLayer2Nodes = routerset_new(); + for (i = 0; i < 10 ; i += 2) { + node_t *vanguard_node = smartlist_get(big_fake_net_nodes, i); + tt_assert(vanguard_node->is_possible_guard); + routerset_parse(options->HSLayer2Nodes, vanguard_node->rs->nickname, "l2"); + } + /* also add some nodes to vanguard L3 list + * (L2 list and L3 list should be disjoint for this test to work) */ + options->HSLayer3Nodes = routerset_new(); + for (i = 10; i < 20 ; i += 2) { + node_t *vanguard_node = smartlist_get(big_fake_net_nodes, i); + tt_assert(vanguard_node->is_possible_guard); + routerset_parse(options->HSLayer3Nodes, vanguard_node->rs->nickname, "l3"); + } + + done: + ; +} + +/** Test to ensure that vanguard path selection works properly. Ensures that + * default vanguard circuits are 4 hops, and that path selection works + * correctly given the vanguard settings. */ +static void +test_entry_guard_vanguard_path_selection(void *arg) +{ + (void) arg; + + int retval; + + /* Enable entry guards */ + or_options_t *options = get_options_mutable(); + options->UseEntryGuards = 1; + + /* XXX disables /16 check */ + options->EnforceDistinctSubnets = 0; + + /* Setup our vanguard list */ + helper_setup_vanguard_list(options); + + /* Create our circuit */ + circuit_t *circ = dummy_origin_circuit_new(30); + origin_circuit_t *oc = TO_ORIGIN_CIRCUIT(circ); + oc->build_state = tor_malloc_zero(sizeof(cpath_build_state_t)); + oc->build_state->is_internal = 1; + + /* Switch circuit purpose to vanguards */ + circ->purpose = CIRCUIT_PURPOSE_HS_VANGUARDS; + + /* First pick the exit and pin it on the build_state */ + tt_int_op(oc->build_state->desired_path_len, OP_EQ, 0); + retval = onion_pick_cpath_exit(oc, NULL, 0); + tt_int_op(retval, OP_EQ, 0); + + /* Ensure that vanguards make 4-hop circuits by default */ + tt_int_op(oc->build_state->desired_path_len, OP_EQ, 4); + + /* Extend path as many times as needed to have complete circ. */ + helper_extend_circuit_path_n_times(oc, oc->build_state->desired_path_len); + + /* Test that the cpath linked list is set correctly. */ + crypt_path_t *l1_node = oc->cpath; + crypt_path_t *l2_node = l1_node->next; + crypt_path_t *l3_node = l2_node->next; + crypt_path_t *l4_node = l3_node->next; + crypt_path_t *l1_node_again = l4_node->next; + tt_ptr_op(l1_node, OP_EQ, l1_node_again); + + /* Test that L2 is indeed HSLayer2Node */ + retval = routerset_contains_extendinfo(options->HSLayer2Nodes, + l2_node->extend_info); + tt_int_op(retval, OP_EQ, 4); + /* test that L3 node is _not_ contained in HSLayer2Node */ + retval = routerset_contains_extendinfo(options->HSLayer2Nodes, + l3_node->extend_info); + tt_int_op(retval, OP_LT, 4); + + /* Test that L3 is indeed HSLayer3Node */ + retval = routerset_contains_extendinfo(options->HSLayer3Nodes, + l3_node->extend_info); + tt_int_op(retval, OP_EQ, 4); + /* test that L2 node is _not_ contained in HSLayer3Node */ + retval = routerset_contains_extendinfo(options->HSLayer3Nodes, + l2_node->extend_info); + tt_int_op(retval, OP_LT, 4); + + /* TODO: Test that L1 can be the same as exit. To test this we need start + enforcing EnforceDistinctSubnets again, which means that we need to give + each test node a different address which currently breaks some tests. */ + + done: + circuit_free_(circ); +} + static const struct testcase_setup_t big_fake_network = { big_fake_network_setup, big_fake_network_cleanup }; @@ -2832,6 +3027,8 @@ struct testcase_t entrynodes_tests[] = { test_entry_guard_parse_from_state_broken, TT_FORK, NULL, NULL }, { "get_guard_selection_by_name", test_entry_guard_get_guard_selection_by_name, TT_FORK, NULL, NULL }, + { "number_of_primaries", + test_entry_guard_number_of_primaries, TT_FORK, NULL, NULL }, BFN_TEST(choose_selection_initial), BFN_TEST(add_single_guard), BFN_TEST(node_filter), @@ -2854,6 +3051,8 @@ struct testcase_t entrynodes_tests[] = { BFN_TEST(select_and_cancel), BFN_TEST(drop_guards), BFN_TEST(outdated_dirserver_exclusion), + BFN_TEST(basic_path_selection), + BFN_TEST(vanguard_path_selection), UPGRADE_TEST(upgrade_a_circuit, "c1-done c2-done"), UPGRADE_TEST(upgrade_blocked_by_live_primary_guards, "c1-done c2-done"), diff --git a/src/test/test_extorport.c b/src/test/test_extorport.c index cadef257f1..e05342cb8a 100644 --- a/src/test/test_extorport.c +++ b/src/test/test_extorport.c @@ -10,6 +10,7 @@ #include "connection_or.h" #include "config.h" #include "control.h" +#include "crypto_rand.h" #include "ext_orport.h" #include "main.h" #include "test.h" diff --git a/src/test/test_geoip.c b/src/test/test_geoip.c new file mode 100644 index 0000000000..6f849f436b --- /dev/null +++ b/src/test/test_geoip.c @@ -0,0 +1,578 @@ +/* Copyright (c) 2001-2004, Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" + +/* These macros pull in declarations for some functions and structures that + * are typically file-private. */ +#define GEOIP_PRIVATE +#include "or.h" +#include "config.h" +#include "geoip.h" +#include "test.h" + + /* Record odd numbered fake-IPs using ipv6, even numbered fake-IPs + * using ipv4. Since our fake geoip database is the same between + * ipv4 and ipv6, we should get the same result no matter which + * address family we pick for each IP. */ +#define SET_TEST_ADDRESS(i) do { \ + if ((i) & 1) { \ + SET_TEST_IPV6(i); \ + tor_addr_from_in6(&addr, &in6); \ + } else { \ + tor_addr_from_ipv4h(&addr, (uint32_t) i); \ + } \ + } while (0) + + /* Make sure that country ID actually works. */ +#define SET_TEST_IPV6(i) \ + do { \ + set_uint32(in6.s6_addr + 12, htonl((uint32_t) (i))); \ + } while (0) +#define CHECK_COUNTRY(country, val) do { \ + /* test ipv4 country lookup */ \ + tt_str_op(country, OP_EQ, \ + geoip_get_country_name(geoip_get_country_by_ipv4(val))); \ + /* test ipv6 country lookup */ \ + SET_TEST_IPV6(val); \ + tt_str_op(country, OP_EQ, \ + geoip_get_country_name(geoip_get_country_by_ipv6(&in6))); \ + } while (0) + +/** Run unit tests for GeoIP code. */ +static void +test_geoip(void *arg) +{ + int i, j; + time_t now = 1281533250; /* 2010-08-11 13:27:30 UTC */ + char *s = NULL, *v = NULL; + const char *bridge_stats_1 = + "bridge-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "bridge-ips zz=24,xy=8\n" + "bridge-ip-versions v4=16,v6=16\n" + "bridge-ip-transports <OR>=24\n", + *dirreq_stats_1 = + "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "dirreq-v3-ips ab=8\n" + "dirreq-v3-reqs ab=8\n" + "dirreq-v3-resp ok=0,not-enough-sigs=0,unavailable=0,not-found=0," + "not-modified=0,busy=0\n" + "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n" + "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0\n", + *dirreq_stats_2 = + "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "dirreq-v3-ips \n" + "dirreq-v3-reqs \n" + "dirreq-v3-resp ok=0,not-enough-sigs=0,unavailable=0,not-found=0," + "not-modified=0,busy=0\n" + "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n" + "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0\n", + *dirreq_stats_3 = + "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "dirreq-v3-ips \n" + "dirreq-v3-reqs \n" + "dirreq-v3-resp ok=8,not-enough-sigs=0,unavailable=0,not-found=0," + "not-modified=0,busy=0\n" + "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n" + "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0\n", + *dirreq_stats_4 = + "dirreq-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "dirreq-v3-ips \n" + "dirreq-v3-reqs \n" + "dirreq-v3-resp ok=8,not-enough-sigs=0,unavailable=0,not-found=0," + "not-modified=0,busy=0\n" + "dirreq-v3-direct-dl complete=0,timeout=0,running=0\n" + "dirreq-v3-tunneled-dl complete=0,timeout=0,running=4\n", + *entry_stats_1 = + "entry-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "entry-ips ab=8\n", + *entry_stats_2 = + "entry-stats-end 2010-08-12 13:27:30 (86400 s)\n" + "entry-ips \n"; + tor_addr_t addr; + struct in6_addr in6; + + /* Populate the DB a bit. Add these in order, since we can't do the final + * 'sort' step. These aren't very good IP addresses, but they're perfectly + * fine uint32_t values. */ + (void)arg; + tt_int_op(0,OP_EQ, geoip_parse_entry("10,50,AB", AF_INET)); + tt_int_op(0,OP_EQ, geoip_parse_entry("52,90,XY", AF_INET)); + tt_int_op(0,OP_EQ, geoip_parse_entry("95,100,AB", AF_INET)); + tt_int_op(0,OP_EQ, geoip_parse_entry("\"105\",\"140\",\"ZZ\"", AF_INET)); + tt_int_op(0,OP_EQ, geoip_parse_entry("\"150\",\"190\",\"XY\"", AF_INET)); + tt_int_op(0,OP_EQ, geoip_parse_entry("\"200\",\"250\",\"AB\"", AF_INET)); + + /* Populate the IPv6 DB equivalently with fake IPs in the same range */ + tt_int_op(0,OP_EQ, geoip_parse_entry("::a,::32,AB", AF_INET6)); + tt_int_op(0,OP_EQ, geoip_parse_entry("::34,::5a,XY", AF_INET6)); + tt_int_op(0,OP_EQ, geoip_parse_entry("::5f,::64,AB", AF_INET6)); + tt_int_op(0,OP_EQ, geoip_parse_entry("::69,::8c,ZZ", AF_INET6)); + tt_int_op(0,OP_EQ, geoip_parse_entry("::96,::be,XY", AF_INET6)); + tt_int_op(0,OP_EQ, geoip_parse_entry("::c8,::fa,AB", AF_INET6)); + + /* We should have 4 countries: ??, ab, xy, zz. */ + tt_int_op(4,OP_EQ, geoip_get_n_countries()); + memset(&in6, 0, sizeof(in6)); + + CHECK_COUNTRY("??", 3); + CHECK_COUNTRY("ab", 32); + CHECK_COUNTRY("??", 5); + CHECK_COUNTRY("??", 51); + CHECK_COUNTRY("xy", 150); + CHECK_COUNTRY("xy", 190); + CHECK_COUNTRY("??", 2000); + + tt_int_op(0,OP_EQ, geoip_get_country_by_ipv4(3)); + SET_TEST_IPV6(3); + tt_int_op(0,OP_EQ, geoip_get_country_by_ipv6(&in6)); + + get_options_mutable()->BridgeRelay = 1; + get_options_mutable()->BridgeRecordUsageByCountry = 1; + /* Put 9 observations in AB... */ + for (i=32; i < 40; ++i) { + SET_TEST_ADDRESS(i); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-7200); + } + SET_TEST_ADDRESS(225); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-7200); + /* and 3 observations in XY, several times. */ + for (j=0; j < 10; ++j) + for (i=52; i < 55; ++i) { + SET_TEST_ADDRESS(i); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-3600); + } + /* and 17 observations in ZZ... */ + for (i=110; i < 127; ++i) { + SET_TEST_ADDRESS(i); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now); + } + geoip_get_client_history(GEOIP_CLIENT_CONNECT, &s, &v); + tt_assert(s); + tt_assert(v); + tt_str_op("zz=24,ab=16,xy=8",OP_EQ, s); + tt_str_op("v4=16,v6=16",OP_EQ, v); + tor_free(s); + tor_free(v); + + /* Now clear out all the AB observations. */ + geoip_remove_old_clients(now-6000); + geoip_get_client_history(GEOIP_CLIENT_CONNECT, &s, &v); + tt_assert(s); + tt_assert(v); + tt_str_op("zz=24,xy=8",OP_EQ, s); + tt_str_op("v4=16,v6=16",OP_EQ, v); + tor_free(s); + tor_free(v); + + /* Start testing bridge statistics by making sure that we don't output + * bridge stats without initializing them. */ + s = geoip_format_bridge_stats(now + 86400); + tt_ptr_op(s, OP_EQ, NULL); + + /* Initialize stats and generate the bridge-stats history string out of + * the connecting clients added above. */ + geoip_bridge_stats_init(now); + s = geoip_format_bridge_stats(now + 86400); + tt_assert(s); + tt_str_op(bridge_stats_1,OP_EQ, s); + tor_free(s); + + /* Stop collecting bridge stats and make sure we don't write a history + * string anymore. */ + geoip_bridge_stats_term(); + s = geoip_format_bridge_stats(now + 86400); + tt_ptr_op(s, OP_EQ, NULL); + + /* Stop being a bridge and start being a directory mirror that gathers + * directory request statistics. */ + geoip_bridge_stats_term(); + get_options_mutable()->BridgeRelay = 0; + get_options_mutable()->BridgeRecordUsageByCountry = 0; + get_options_mutable()->DirReqStatistics = 1; + + /* Start testing dirreq statistics by making sure that we don't collect + * dirreq stats without initializing them. */ + SET_TEST_ADDRESS(100); + geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now); + s = geoip_format_dirreq_stats(now + 86400); + tt_ptr_op(s, OP_EQ, NULL); + + /* Initialize stats, note one connecting client, and generate the + * dirreq-stats history string. */ + geoip_dirreq_stats_init(now); + SET_TEST_ADDRESS(100); + geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now); + s = geoip_format_dirreq_stats(now + 86400); + tt_str_op(dirreq_stats_1,OP_EQ, s); + tor_free(s); + + /* Stop collecting stats, add another connecting client, and ensure we + * don't generate a history string. */ + geoip_dirreq_stats_term(); + SET_TEST_ADDRESS(101); + geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now); + s = geoip_format_dirreq_stats(now + 86400); + tt_ptr_op(s, OP_EQ, NULL); + + /* Re-start stats, add a connecting client, reset stats, and make sure + * that we get an all empty history string. */ + geoip_dirreq_stats_init(now); + SET_TEST_ADDRESS(100); + geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS, &addr, NULL, now); + geoip_reset_dirreq_stats(now); + s = geoip_format_dirreq_stats(now + 86400); + tt_str_op(dirreq_stats_2,OP_EQ, s); + tor_free(s); + + /* Note a successful network status response and make sure that it + * appears in the history string. */ + geoip_note_ns_response(GEOIP_SUCCESS); + s = geoip_format_dirreq_stats(now + 86400); + tt_str_op(dirreq_stats_3,OP_EQ, s); + tor_free(s); + + /* Start a tunneled directory request. */ + geoip_start_dirreq((uint64_t) 1, 1024, DIRREQ_TUNNELED); + s = geoip_format_dirreq_stats(now + 86400); + tt_str_op(dirreq_stats_4,OP_EQ, s); + tor_free(s); + + /* Stop collecting directory request statistics and start gathering + * entry stats. */ + geoip_dirreq_stats_term(); + get_options_mutable()->DirReqStatistics = 0; + get_options_mutable()->EntryStatistics = 1; + + /* Start testing entry statistics by making sure that we don't collect + * anything without initializing entry stats. */ + SET_TEST_ADDRESS(100); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now); + s = geoip_format_entry_stats(now + 86400); + tt_ptr_op(s, OP_EQ, NULL); + + /* Initialize stats, note one connecting client, and generate the + * entry-stats history string. */ + geoip_entry_stats_init(now); + SET_TEST_ADDRESS(100); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now); + s = geoip_format_entry_stats(now + 86400); + tt_str_op(entry_stats_1,OP_EQ, s); + tor_free(s); + + /* Stop collecting stats, add another connecting client, and ensure we + * don't generate a history string. */ + geoip_entry_stats_term(); + SET_TEST_ADDRESS(101); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now); + s = geoip_format_entry_stats(now + 86400); + tt_ptr_op(s, OP_EQ, NULL); + + /* Re-start stats, add a connecting client, reset stats, and make sure + * that we get an all empty history string. */ + geoip_entry_stats_init(now); + SET_TEST_ADDRESS(100); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now); + geoip_reset_entry_stats(now); + s = geoip_format_entry_stats(now + 86400); + tt_str_op(entry_stats_2,OP_EQ, s); + tor_free(s); + + /* Test the OOM handler. Add a client, run the OOM. */ + geoip_entry_stats_init(now); + SET_TEST_ADDRESS(100); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, + now - (12 * 60 * 60)); + /* We've seen this 12 hours ago. Run the OOM, it should clean the entry + * because it is above the minimum cutoff of 4 hours. */ + size_t bytes_removed = geoip_client_cache_handle_oom(now, 1000); + tt_size_op(bytes_removed, OP_GT, 0); + + /* Do it again but this time with an entry with a lower cutoff. */ + geoip_entry_stats_init(now); + SET_TEST_ADDRESS(100); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, + now - (3 * 60 * 60)); + bytes_removed = geoip_client_cache_handle_oom(now, 1000); + tt_size_op(bytes_removed, OP_EQ, 0); + + /* Stop collecting entry statistics. */ + geoip_entry_stats_term(); + get_options_mutable()->EntryStatistics = 0; + + done: + tor_free(s); + tor_free(v); +} + +static void +test_geoip_with_pt(void *arg) +{ + time_t now = 1281533250; /* 2010-08-11 13:27:30 UTC */ + char *s = NULL; + int i; + tor_addr_t addr; + struct in6_addr in6; + + (void)arg; + get_options_mutable()->BridgeRelay = 1; + get_options_mutable()->BridgeRecordUsageByCountry = 1; + + memset(&in6, 0, sizeof(in6)); + + /* No clients seen yet. */ + s = geoip_get_transport_history(); + tor_assert(!s); + + /* 4 connections without a pluggable transport */ + for (i=0; i < 4; ++i) { + SET_TEST_ADDRESS(i); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, NULL, now-7200); + } + + /* 9 connections with "alpha" */ + for (i=4; i < 13; ++i) { + SET_TEST_ADDRESS(i); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "alpha", now-7200); + } + + /* one connection with "beta" */ + SET_TEST_ADDRESS(13); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "beta", now-7200); + + /* 14 connections with "charlie" */ + for (i=14; i < 28; ++i) { + SET_TEST_ADDRESS(i); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "charlie", now-7200); + } + + /* 131 connections with "ddr" */ + for (i=28; i < 159; ++i) { + SET_TEST_ADDRESS(i); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "ddr", now-7200); + } + + /* 8 connections with "entropy" */ + for (i=159; i < 167; ++i) { + SET_TEST_ADDRESS(i); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "entropy", now-7200); + } + + /* 2 connections from the same IP with two different transports. */ + SET_TEST_ADDRESS(++i); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "fire", now-7200); + geoip_note_client_seen(GEOIP_CLIENT_CONNECT, &addr, "google", now-7200); + + /* Test the transport history string. */ + s = geoip_get_transport_history(); + tor_assert(s); + tt_str_op(s,OP_EQ, "<OR>=8,alpha=16,beta=8,charlie=16,ddr=136," + "entropy=8,fire=8,google=8"); + + /* Stop collecting entry statistics. */ + geoip_entry_stats_term(); + get_options_mutable()->EntryStatistics = 0; + + done: + tor_free(s); +} + +#undef SET_TEST_ADDRESS +#undef SET_TEST_IPV6 +#undef CHECK_COUNTRY + +static const char GEOIP_CONTENT[] = + "134445936,134445939,MP\n" + "134445940,134447103,GU\n" + "134447104,134738943,US\n" + "134738944,134739199,CA\n" + "134739200,135192575,US\n" + "135192576,135200767,MX\n" + "135200768,135430143,US\n" + "135430144,135430399,CA\n" + "135430400,135432191,US\n"; + +static void +test_geoip_load_file(void *arg) +{ + (void)arg; + char *contents = NULL; + char *dhex = NULL; + + /* A nonexistant filename should fail. */ + tt_int_op(-1, OP_EQ, + geoip_load_file(AF_INET, "/you/did/not/put/a/file/here/I/hope")); + + /* We start out with only "Ningunpartia" in the database. */ + tt_int_op(1, OP_EQ, geoip_get_n_countries()); + tt_str_op("??", OP_EQ, geoip_get_country_name(0)); + /* Any lookup attempt should say "-1" because we have no info */ + tt_int_op(-1, OP_EQ, geoip_get_country_by_ipv4(0x01020304)); + /* There should be no 'digest' for a nonexistant file */ + tt_str_op("0000000000000000000000000000000000000000", OP_EQ, + geoip_db_digest(AF_INET)); + + const char *fname = get_fname("geoip"); + tt_int_op(0, OP_EQ, write_str_to_file(fname, GEOIP_CONTENT, 1)); + + int rv = geoip_load_file(AF_INET, fname); + if (rv != 0) { + TT_GRIPE(("Unable to load geoip from %s", escaped(fname))); + } + tt_int_op(0, OP_EQ, rv); + + /* Check that we loaded some countries; this will fail if there are ever + * fewer than 5 countries in our test above. */ + tt_int_op(geoip_get_n_countries(), OP_GE, 5); + + /* Let's see where 8.8.8.8 is. */ + int country = geoip_get_country_by_ipv4(0x08080808); + tt_int_op(country, OP_GE, 1); /* It shouldn't be 'unknown' or 'nowhere' */ + const char *cc = geoip_get_country_name(country); + tt_int_op(strlen(cc), OP_EQ, 2); + + /* The digest should be set.... */ + tt_str_op("0000000000000000000000000000000000000000", OP_NE, + geoip_db_digest(AF_INET)); + + /* And it should be set correctly */ + contents = read_file_to_str(fname, RFTS_BIN, NULL); + uint8_t d[DIGEST_LEN]; + crypto_digest((char*)d, contents, strlen(contents)); + dhex = tor_strdup(hex_str((char*)d, DIGEST_LEN)); + tt_str_op(dhex, OP_EQ, geoip_db_digest(AF_INET)); + + /* Make sure geoip_free_all() works. */ + geoip_free_all(); + tt_int_op(1, OP_EQ, geoip_get_n_countries()); + tt_str_op("??", OP_EQ, geoip_get_country_name(0)); + tt_int_op(-1, OP_EQ, geoip_get_country_by_ipv4(0x01020304)); + tt_str_op("0000000000000000000000000000000000000000", OP_EQ, + geoip_db_digest(AF_INET)); // <--- nick bets this will fail. + + done: + tor_free(contents); + tor_free(dhex); +} + +static void +test_geoip6_load_file(void *arg) +{ + (void)arg; + struct in6_addr iaddr6; + char *contents = NULL; + char *dhex = NULL; + + /* A nonexistant filename should fail. */ + tt_int_op(-1, OP_EQ, + geoip_load_file(AF_INET6, "/you/did/not/put/a/file/here/I/hope")); + + /* Any lookup attempt should say "-1" because we have no info */ + tor_inet_pton(AF_INET6, "2001:4860:4860::8888", &iaddr6); + tt_int_op(-1, OP_EQ, geoip_get_country_by_ipv6(&iaddr6)); + + /* Load geiop6 file */ + const char *fname6 = get_fname("geoip6"); + const char CONTENT[] = + "2001:4830:6010::,2001:4830:601f:ffff:ffff:ffff:ffff:ffff,GB\n" + "2001:4830:6020::,2001:4830:ffff:ffff:ffff:ffff:ffff:ffff,US\n" + "2001:4838::,2001:4838:ffff:ffff:ffff:ffff:ffff:ffff,US\n" + "2001:4840::,2001:4840:ffff:ffff:ffff:ffff:ffff:ffff,XY\n" + "2001:4848::,2001:4848:ffff:ffff:ffff:ffff:ffff:ffff,ZD\n" + "2001:4850::,2001:4850:ffff:ffff:ffff:ffff:ffff:ffff,RO\n" + "2001:4858::,2001:4858:ffff:ffff:ffff:ffff:ffff:ffff,TC\n" + "2001:4860::,2001:4860:ffff:ffff:ffff:ffff:ffff:ffff,US\n" + "2001:4868::,2001:4868:ffff:ffff:ffff:ffff:ffff:ffff,US\n" + "2001:4870::,2001:4871:ffff:ffff:ffff:ffff:ffff:ffff,NB\n" + "2001:4878::,2001:4878:128:ffff:ffff:ffff:ffff:ffff,US\n" + "2001:4878:129::,2001:4878:129:ffff:ffff:ffff:ffff:ffff,CR\n" + "2001:4878:12a::,2001:4878:203:ffff:ffff:ffff:ffff:ffff,US\n" + "2001:4878:204::,2001:4878:204:ffff:ffff:ffff:ffff:ffff,DE\n" + "2001:4878:205::,2001:4878:214:ffff:ffff:ffff:ffff:ffff,US\n"; + tt_int_op(0, OP_EQ, write_str_to_file(fname6, CONTENT, 1)); + + tt_int_op(0, OP_EQ, geoip_load_file(AF_INET6, fname6)); + + /* Check that we loaded some countries; this will fail if there are ever + * fewer than 5 countries in our test data above. */ + tt_int_op(geoip_get_n_countries(), OP_GE, 5); + + /* Let's see where 2001:4860:4860::8888 (google dns) is. */ + const char *caddr6 = "2001:4860:4860::8888"; + tor_inet_pton(AF_INET6, caddr6, &iaddr6); + int country6 = geoip_get_country_by_ipv6(&iaddr6); + tt_int_op(country6, OP_GE, 1); + + const char *cc6 = geoip_get_country_name(country6); + tt_int_op(strlen(cc6), OP_EQ, 2); + + /* The digest should be set.... */ + tt_str_op("0000000000000000000000000000000000000000", OP_NE, + geoip_db_digest(AF_INET6)); + + /* And it should be set correctly */ + contents = read_file_to_str(fname6, RFTS_BIN, NULL); + uint8_t d[DIGEST_LEN]; + crypto_digest((char*)d, contents, strlen(contents)); + dhex = tor_strdup(hex_str((char*)d, DIGEST_LEN)); + tt_str_op(dhex, OP_EQ, geoip_db_digest(AF_INET6)); + + /* Make sure geoip_free_all() works. */ + geoip_free_all(); + tt_int_op(1, OP_EQ, geoip_get_n_countries()); + tt_str_op("??", OP_EQ, geoip_get_country_name(0)); + tor_inet_pton(AF_INET6, "::1:2:3:4", &iaddr6); + tt_int_op(-1, OP_EQ, geoip_get_country_by_ipv6(&iaddr6)); + tt_str_op("0000000000000000000000000000000000000000", OP_EQ, + geoip_db_digest(AF_INET6)); + + done: + tor_free(contents); + tor_free(dhex); +} + +static void +test_geoip_load_2nd_file(void *arg) +{ + (void)arg; + + char *fname_geoip = tor_strdup(get_fname("geoip_data")); + char *fname_empty = tor_strdup(get_fname("geoip_empty")); + + tt_int_op(0, OP_EQ, write_str_to_file(fname_geoip, GEOIP_CONTENT, 1)); + tt_int_op(0, OP_EQ, write_str_to_file(fname_empty, "\n", 1)); + + /* Load 1st geoip file */ + tt_int_op(0, OP_EQ, geoip_load_file(AF_INET, fname_geoip)); + + /* Load 2nd geoip (empty) file */ + /* It has to be the same IP address family */ + tt_int_op(0, OP_EQ, geoip_load_file(AF_INET, fname_empty)); + + /* Check that there is no geoip information for 8.8.8.8, */ + /* since loading the empty 2nd file should have delete it. */ + int country = geoip_get_country_by_ipv4(0x08080808); + tt_int_op(country, OP_EQ, 0); + + done: + tor_free(fname_geoip); + tor_free(fname_empty); +} + +#define ENT(name) \ + { #name, test_ ## name , 0, NULL, NULL } +#define FORK(name) \ + { #name, test_ ## name , TT_FORK, NULL, NULL } + +struct testcase_t geoip_tests[] = { + { "geoip", test_geoip, TT_FORK, NULL, NULL }, + { "geoip_with_pt", test_geoip_with_pt, TT_FORK, NULL, NULL }, + { "load_file", test_geoip_load_file, TT_FORK, NULL, NULL }, + { "load_file6", test_geoip6_load_file, TT_FORK, NULL, NULL }, + { "load_2nd_file", test_geoip_load_2nd_file, TT_FORK, NULL, NULL }, + + END_OF_TESTCASES +}; + diff --git a/src/test/test_helpers.c b/src/test/test_helpers.c index 0da9cf64d0..1db5e9064f 100644 --- a/src/test/test_helpers.c +++ b/src/test/test_helpers.c @@ -18,6 +18,7 @@ #include "config.h" #include "confparse.h" #include "connection.h" +#include "crypto_rand.h" #include "main.h" #include "nodelist.h" #include "relay.h" @@ -33,7 +34,6 @@ DISABLE_GCC_WARNING(overlength-strings) * at large. */ #endif #include "test_descriptors.inc" -#include "or.h" #include "circuitlist.h" #ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS ENABLE_GCC_WARNING(overlength-strings) @@ -156,7 +156,7 @@ mock_tor_addr_lookup__fail_on_bad_addrs(const char *name, /* Helper for test_conn_get_connection() */ static int -fake_close_socket(evutil_socket_t sock) +fake_close_socket(tor_socket_t sock) { (void)sock; return 0; diff --git a/src/test/test_hs_cell.c b/src/test/test_hs_cell.c index aed28d3bd2..8e15184c2a 100644 --- a/src/test/test_hs_cell.c +++ b/src/test/test_hs_cell.c @@ -14,6 +14,7 @@ #include "log_test_helpers.h" #include "crypto_ed25519.h" +#include "crypto_rand.h" #include "hs_cell.h" #include "hs_intropoint.h" #include "hs_service.h" diff --git a/src/test/test_hs_client.c b/src/test/test_hs_client.c index 7ee7210bc9..50dca588ed 100644 --- a/src/test/test_hs_client.c +++ b/src/test/test_hs_client.c @@ -213,12 +213,12 @@ test_e2e_rend_circuit_setup_legacy(void *arg) tt_int_op(retval, OP_EQ, 1); /* Check the digest algo */ - tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->f_digest), + tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.f_digest), OP_EQ, DIGEST_SHA1); - tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->b_digest), + tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.b_digest), OP_EQ, DIGEST_SHA1); - tt_assert(or_circ->cpath->f_crypto); - tt_assert(or_circ->cpath->b_crypto); + tt_assert(or_circ->cpath->crypto.f_crypto); + tt_assert(or_circ->cpath->crypto.b_crypto); /* Ensure that circ purpose was changed */ tt_int_op(or_circ->base_.purpose, OP_EQ, CIRCUIT_PURPOSE_C_REND_JOINED); @@ -283,12 +283,12 @@ test_e2e_rend_circuit_setup(void *arg) tt_int_op(retval, OP_EQ, 1); /* Check that the crypt path has prop224 algorithm parameters */ - tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->f_digest), + tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.f_digest), OP_EQ, DIGEST_SHA3_256); - tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->b_digest), + tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.b_digest), OP_EQ, DIGEST_SHA3_256); - tt_assert(or_circ->cpath->f_crypto); - tt_assert(or_circ->cpath->b_crypto); + tt_assert(or_circ->cpath->crypto.f_crypto); + tt_assert(or_circ->cpath->crypto.b_crypto); /* Ensure that circ purpose was changed */ tt_int_op(or_circ->base_.purpose, OP_EQ, CIRCUIT_PURPOSE_C_REND_JOINED); @@ -397,21 +397,25 @@ test_client_pick_intro(void *arg) } SMARTLIST_FOREACH_END(ip); /* Try to get a random intro: Should return the chosen one! */ - extend_info_t *ip = client_get_random_intro(&service_kp.pubkey); - tor_assert(ip); - tt_assert(!tor_mem_is_zero((char*)ip->identity_digest, DIGEST_LEN)); - tt_mem_op(ip->identity_digest, OP_EQ, chosen_intro_ei->identity_digest, - DIGEST_LEN); + /* (We try several times, to make sure this behavior is consistent, and to + * cover the different cases of client_get_random_intro().) */ + for (int i = 0; i < 64; ++i) { + extend_info_t *ip = client_get_random_intro(&service_kp.pubkey); + tor_assert(ip); + tt_assert(!tor_mem_is_zero((char*)ip->identity_digest, DIGEST_LEN)); + tt_mem_op(ip->identity_digest, OP_EQ, chosen_intro_ei->identity_digest, + DIGEST_LEN); + extend_info_free(ip); + } extend_info_free(chosen_intro_ei); - extend_info_free(ip); /* Now also mark the chosen one as failed: See that we can't get any intro points anymore. */ hs_cache_client_intro_state_note(&service_kp.pubkey, &chosen_intro_point->auth_key_cert->signed_key, INTRO_POINT_FAILURE_TIMEOUT); - ip = client_get_random_intro(&service_kp.pubkey); + extend_info_t *ip = client_get_random_intro(&service_kp.pubkey); tor_assert(!ip); } diff --git a/src/test/test_hs_common.c b/src/test/test_hs_common.c index 16803dbd16..7348eb746c 100644 --- a/src/test/test_hs_common.c +++ b/src/test/test_hs_common.c @@ -17,19 +17,21 @@ #include "hs_test_helpers.h" #include "connection_edge.h" +#include "crypto_rand.h" #include "hs_common.h" #include "hs_client.h" #include "hs_service.h" #include "config.h" #include "networkstatus.h" #include "directory.h" -#include "dirvote.h" +#include "dirauth/dirvote.h" #include "nodelist.h" #include "routerlist.h" #include "statefile.h" #include "circuitlist.h" -#include "shared_random.h" +#include "dirauth/shared_random.h" #include "util.h" +#include "voting_schedule.h" /** Test the validation of HS v3 addresses */ static void @@ -364,11 +366,8 @@ mock_networkstatus_get_live_consensus(time_t now) static void test_responsible_hsdirs(void *arg) { - time_t now = approx_time(); smartlist_t *responsible_dirs = smartlist_new(); networkstatus_t *ns = NULL; - int retval; - (void) arg; hs_init(); @@ -390,12 +389,12 @@ test_responsible_hsdirs(void *arg) helper_add_hsdir_to_networkstatus(ns, 3, "spyro", 0); } - ed25519_keypair_t kp; - retval = ed25519_keypair_generate(&kp, 0); - tt_int_op(retval, OP_EQ , 0); + /* Use a fixed time period and pub key so we always take the same path */ + ed25519_public_key_t pubkey; + uint64_t time_period_num = 17653; // 2 May, 2018, 14:00. + memset(&pubkey, 42, sizeof(pubkey)); - uint64_t time_period_num = hs_get_time_period_num(now); - hs_get_responsible_hsdirs(&kp.pubkey, time_period_num, + hs_get_responsible_hsdirs(&pubkey, time_period_num, 0, 0, responsible_dirs); /* Make sure that we only found 2 responsible HSDirs. @@ -817,7 +816,7 @@ test_time_between_tp_and_srv(void *arg) tt_int_op(ret, OP_EQ, 0); ret = parse_rfc1123_time("Sat, 26 Oct 1985 01:00:00 UTC", &ns.fresh_until); tt_int_op(ret, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), ns.valid_after); + voting_schedule_recalculate_timing(get_options(), ns.valid_after); ret = hs_in_period_between_tp_and_srv(&ns, 0); tt_int_op(ret, OP_EQ, 0); @@ -825,7 +824,7 @@ test_time_between_tp_and_srv(void *arg) tt_int_op(ret, OP_EQ, 0); ret = parse_rfc1123_time("Sat, 26 Oct 1985 12:00:00 UTC", &ns.fresh_until); tt_int_op(ret, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), ns.valid_after); + voting_schedule_recalculate_timing(get_options(), ns.valid_after); ret = hs_in_period_between_tp_and_srv(&ns, 0); tt_int_op(ret, OP_EQ, 0); @@ -833,7 +832,7 @@ test_time_between_tp_and_srv(void *arg) tt_int_op(ret, OP_EQ, 0); ret = parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC", &ns.fresh_until); tt_int_op(ret, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), ns.valid_after); + voting_schedule_recalculate_timing(get_options(), ns.valid_after); ret = hs_in_period_between_tp_and_srv(&ns, 0); tt_int_op(ret, OP_EQ, 1); @@ -841,7 +840,7 @@ test_time_between_tp_and_srv(void *arg) tt_int_op(ret, OP_EQ, 0); ret = parse_rfc1123_time("Sat, 27 Oct 1985 00:00:00 UTC", &ns.fresh_until); tt_int_op(ret, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), ns.valid_after); + voting_schedule_recalculate_timing(get_options(), ns.valid_after); ret = hs_in_period_between_tp_and_srv(&ns, 0); tt_int_op(ret, OP_EQ, 1); @@ -849,7 +848,7 @@ test_time_between_tp_and_srv(void *arg) tt_int_op(ret, OP_EQ, 0); ret = parse_rfc1123_time("Sat, 27 Oct 1985 01:00:00 UTC", &ns.fresh_until); tt_int_op(ret, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), ns.valid_after); + voting_schedule_recalculate_timing(get_options(), ns.valid_after); ret = hs_in_period_between_tp_and_srv(&ns, 0); tt_int_op(ret, OP_EQ, 0); @@ -1336,7 +1335,8 @@ run_reachability_scenario(const reachability_cfg_t *cfg, int num_scenario) &mock_service_ns->valid_until); set_consensus_times(cfg->service_valid_until, &mock_service_ns->fresh_until); - dirvote_recalculate_timing(get_options(), mock_service_ns->valid_after); + voting_schedule_recalculate_timing(get_options(), + mock_service_ns->valid_after); /* Set client consensus time. */ set_consensus_times(cfg->client_valid_after, &mock_client_ns->valid_after); @@ -1344,7 +1344,8 @@ run_reachability_scenario(const reachability_cfg_t *cfg, int num_scenario) &mock_client_ns->valid_until); set_consensus_times(cfg->client_valid_until, &mock_client_ns->fresh_until); - dirvote_recalculate_timing(get_options(), mock_client_ns->valid_after); + voting_schedule_recalculate_timing(get_options(), + mock_client_ns->valid_after); /* New time period checks for this scenario. */ tt_int_op(hs_in_period_between_tp_and_srv(mock_service_ns, 0), OP_EQ, @@ -1568,7 +1569,7 @@ helper_set_consensus_and_system_time(networkstatus_t *ns, int position) } else { tt_assert(0); } - dirvote_recalculate_timing(get_options(), ns->valid_after); + voting_schedule_recalculate_timing(get_options(), ns->valid_after); /* Set system time: pretend to be just 2 minutes before consensus expiry */ real_time = ns->valid_until - 120; diff --git a/src/test/test_hs_control.c b/src/test/test_hs_control.c index 207a55de6d..308843e9b8 100644 --- a/src/test/test_hs_control.c +++ b/src/test/test_hs_control.c @@ -76,9 +76,8 @@ mock_node_get_by_id(const char *digest) { static node_t node; memcpy(node.identity, digest, DIGEST_LEN); - node.hsdir_index = tor_malloc_zero(sizeof(hsdir_index_t)); - memset(node.hsdir_index->fetch, 'C', DIGEST256_LEN); - memset(node.hsdir_index->store_first, 'D', DIGEST256_LEN); + memset(node.hsdir_index.fetch, 'C', DIGEST256_LEN); + memset(node.hsdir_index.store_first, 'D', DIGEST256_LEN); return &node; } diff --git a/src/test/test_hs_descriptor.c b/src/test/test_hs_descriptor.c index 8772461f90..14f1a664e7 100644 --- a/src/test/test_hs_descriptor.c +++ b/src/test/test_hs_descriptor.c @@ -9,6 +9,8 @@ #define HS_DESCRIPTOR_PRIVATE #include "crypto_ed25519.h" +#include "crypto_digest.h" +#include "crypto_rand.h" #include "ed25519_cert.h" #include "or.h" #include "hs_descriptor.h" diff --git a/src/test/test_hs_intropoint.c b/src/test/test_hs_intropoint.c index ec4dcb4705..faa14d9015 100644 --- a/src/test/test_hs_intropoint.c +++ b/src/test/test_hs_intropoint.c @@ -13,8 +13,7 @@ #include "test.h" #include "log_test_helpers.h" -#include "crypto.h" -#include "log_test_helpers.h" +#include "crypto_rand.h" #include "or.h" #include "circuitlist.h" diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c index c1e9f3ced6..7972434d69 100644 --- a/src/test/test_hs_service.c +++ b/src/test/test_hs_service.c @@ -33,13 +33,12 @@ #include "circuitbuild.h" #include "circuitlist.h" #include "circuituse.h" -#include "crypto.h" -#include "dirvote.h" +#include "crypto_rand.h" +#include "dirauth/dirvote.h" #include "networkstatus.h" #include "nodelist.h" #include "relay.h" #include "routerparse.h" - #include "hs_common.h" #include "hs_config.h" #include "hs_ident.h" @@ -51,7 +50,8 @@ #include "main.h" #include "rendservice.h" #include "statefile.h" -#include "shared_random_state.h" +#include "dirauth/shared_random_state.h" +#include "voting_schedule.h" /* Trunnel */ #include "hs/cell_establish_intro.h" @@ -173,12 +173,12 @@ test_e2e_rend_circuit_setup(void *arg) tt_int_op(retval, OP_EQ, 1); /* Check the digest algo */ - tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->f_digest), + tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.f_digest), OP_EQ, DIGEST_SHA3_256); - tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->b_digest), + tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.b_digest), OP_EQ, DIGEST_SHA3_256); - tt_assert(or_circ->cpath->f_crypto); - tt_assert(or_circ->cpath->b_crypto); + tt_assert(or_circ->cpath->crypto.f_crypto); + tt_assert(or_circ->cpath->crypto.b_crypto); /* Ensure that circ purpose was changed */ tt_int_op(or_circ->base_.purpose, OP_EQ, CIRCUIT_PURPOSE_S_REND_JOINED); @@ -492,6 +492,8 @@ test_helper_functions(void *arg) MOCK(node_get_by_id, mock_node_get_by_id); hs_service_init(); + time_t now = time(NULL); + update_approx_time(now); service = helper_create_service(); @@ -551,7 +553,6 @@ test_helper_functions(void *arg) /* Testing can_service_launch_intro_circuit() */ { - time_t now = time(NULL); /* Put the start of the retry period back in time, we should be allowed. * to launch intro circuit. */ service->state.num_intro_circ_launched = 2; @@ -575,7 +576,6 @@ test_helper_functions(void *arg) /* Testing intro_point_should_expire(). */ { - time_t now = time(NULL); /* Just some basic test of the current state. */ tt_u64_op(ip->introduce2_max, OP_GE, INTRO_POINT_MIN_LIFETIME_INTRODUCTIONS); @@ -1057,7 +1057,7 @@ test_rotate_descriptors(void *arg) ret = parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC", &mock_ns.fresh_until); tt_int_op(ret, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), mock_ns.valid_after); + voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after); /* Create a service with a default descriptor and state. It's added to the * global map. */ @@ -1095,7 +1095,7 @@ test_rotate_descriptors(void *arg) ret = parse_rfc1123_time("Sat, 27 Oct 1985 02:00:00 UTC", &mock_ns.fresh_until); tt_int_op(ret, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), mock_ns.valid_after); + voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after); /* Note down what to expect for the next rotation time which is 01:00 + 23h * meaning 00:00:00. */ @@ -1157,7 +1157,7 @@ test_build_update_descriptors(void *arg) ret = parse_rfc1123_time("Sat, 26 Oct 1985 04:00:00 UTC", &mock_ns.fresh_until); tt_int_op(ret, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), mock_ns.valid_after); + voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after); /* Create a service without a current descriptor to trigger a build. */ service = helper_create_service(); @@ -1237,7 +1237,7 @@ test_build_update_descriptors(void *arg) node->is_running = node->is_valid = node->is_fast = node->is_stable = 1; } - /* We have to set thise, or the lack of microdescriptors for these + /* We have to set this, or the lack of microdescriptors for these * nodes will make them unusable. */ get_options_mutable()->UseMicrodescriptors = 0; diff --git a/src/test/test_mainloop.c b/src/test/test_mainloop.c new file mode 100644 index 0000000000..9da8a039dd --- /dev/null +++ b/src/test/test_mainloop.c @@ -0,0 +1,142 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_mainloop.c + * \brief Tests for functions closely related to the Tor main loop + */ + +#include "test.h" +#include "log_test_helpers.h" + +#include "or.h" +#include "main.h" + +static const uint64_t BILLION = 1000000000; + +static void +test_mainloop_update_time_normal(void *arg) +{ + (void)arg; + + monotime_enable_test_mocking(); + /* This is arbitrary */ + uint64_t mt_now = U64_LITERAL(7493289274986); + /* This time is in the past as of when this test was written. */ + time_t now = 1525272090; + monotime_coarse_set_mock_time_nsec(mt_now); + reset_uptime(); + update_current_time(now); + tt_int_op(approx_time(), OP_EQ, now); + tt_int_op(get_uptime(), OP_EQ, 0); + + update_current_time(now); // Same time as before is a no-op. + tt_int_op(get_uptime(), OP_EQ, 0); + + now += 1; + mt_now += BILLION; + monotime_coarse_set_mock_time_nsec(mt_now); + update_current_time(now); + tt_int_op(approx_time(), OP_EQ, now); + tt_int_op(get_uptime(), OP_EQ, 1); + + now += 2; // two-second jump is unremarkable. + mt_now += 2*BILLION; + update_current_time(now); + monotime_coarse_set_mock_time_nsec(mt_now); + tt_int_op(approx_time(), OP_EQ, now); + tt_int_op(get_uptime(), OP_EQ, 3); + + now -= 1; // a one-second hop backwards is also unremarkable. + update_current_time(now); + tt_int_op(approx_time(), OP_EQ, now); // it changes the approx time... + tt_int_op(get_uptime(), OP_EQ, 3); // but it doesn't roll back our uptime + + done: + monotime_disable_test_mocking(); +} + +static void +test_mainloop_update_time_jumps(void *arg) +{ + (void)arg; + + monotime_enable_test_mocking(); + /* This is arbitrary */ + uint64_t mt_now = U64_LITERAL(7493289274986); + /* This time is in the past as of when this test was written. */ + time_t now = 220897152; + monotime_coarse_set_mock_time_nsec(mt_now); + reset_uptime(); + update_current_time(now); + tt_int_op(approx_time(), OP_EQ, now); + tt_int_op(get_uptime(), OP_EQ, 0); + + /* Put some uptime on the clock.. */ + now += 3; + mt_now += 3*BILLION; + monotime_coarse_set_mock_time_nsec(mt_now); + update_current_time(now); + tt_int_op(approx_time(), OP_EQ, now); + tt_int_op(get_uptime(), OP_EQ, 3); + + /* Now try jumping forward and backward, without updating the monotonic + * clock. */ + setup_capture_of_logs(LOG_NOTICE); + now += 1800; + update_current_time(now); + expect_single_log_msg_containing( + "Your system clock just jumped 1800 seconds forward"); + tt_int_op(approx_time(), OP_EQ, now); + tt_int_op(get_uptime(), OP_EQ, 3); // no uptime change. + mock_clean_saved_logs(); + + now -= 600; + update_current_time(now); + expect_single_log_msg_containing( + "Your system clock just jumped 600 seconds backward"); + tt_int_op(approx_time(), OP_EQ, now); + tt_int_op(get_uptime(), OP_EQ, 3); // no uptime change. + mock_clean_saved_logs(); + + /* uptime tracking should go normally now if the clock moves sensibly. */ + now += 2; + mt_now += 2*BILLION; + update_current_time(now); + tt_int_op(approx_time(), OP_EQ, now); + tt_int_op(get_uptime(), OP_EQ, 5); + + /* If we skip forward by a few minutes but the monotonic clock agrees, + * we've just been idle: that counts as not worth warning about. */ + now += 1800; + mt_now += 1800*BILLION; + monotime_coarse_set_mock_time_nsec(mt_now); + update_current_time(now); + expect_no_log_entry(); + tt_int_op(approx_time(), OP_EQ, now); + tt_int_op(get_uptime(), OP_EQ, 5); // this doesn't count to uptime, though. + + /* If we skip forward by a long time, even if the clock agrees, it's + * idnless that counts. */ + now += 4000; + mt_now += 4000*BILLION; + monotime_coarse_set_mock_time_nsec(mt_now); + update_current_time(now); + expect_single_log_msg_containing("Tor has been idle for 4000 seconds"); + tt_int_op(approx_time(), OP_EQ, now); + tt_int_op(get_uptime(), OP_EQ, 5); + + done: + teardown_capture_of_logs(); + monotime_disable_test_mocking(); +} + +#define MAINLOOP_TEST(name) \ + { #name, test_mainloop_## name , TT_FORK, NULL, NULL } + +struct testcase_t mainloop_tests[] = { + MAINLOOP_TEST(update_time_normal), + MAINLOOP_TEST(update_time_jumps), + END_OF_TESTCASES +}; + diff --git a/src/test/test_microdesc.c b/src/test/test_microdesc.c index 59b28f7580..4b168f49ed 100644 --- a/src/test/test_microdesc.c +++ b/src/test/test_microdesc.c @@ -5,7 +5,8 @@ #include "or.h" #include "config.h" -#include "dirvote.h" +#define DIRVOTE_PRIVATE +#include "dirauth/dirvote.h" #include "microdesc.h" #include "networkstatus.h" #include "routerlist.h" @@ -385,25 +386,6 @@ static const char test_ri2[] = "cf34GXHv61XReJF3AlzNHFpbrPOYmowmhrTULKyMqow=\n" "-----END SIGNATURE-----\n"; -static const char test_md_8[] = - "onion-key\n" - "-----BEGIN RSA PUBLIC KEY-----\n" - "MIGJAoGBANBJz8Vldl12aFeSMPLiA4nOetLDN0oxU8bB1SDhO7Uu2zdWYVYAF5J0\n" - "st7WvrVy/jA9v/fsezNAPskBanecHRSkdMTpkcgRPMHE7CTGEwIy1Yp1X4bPgDlC\n" - "VCnbs5Pcts5HnWEYNK7qHDAUn+IlmjOO+pTUY8uyq+GQVz6H9wFlAgMBAAE=\n" - "-----END RSA PUBLIC KEY-----\n" - "p reject 25,119,135-139,445,563,1214,4661-4666,6346-6429,6699,6881-6999\n"; - -static const char test_md_16[] = - "onion-key\n" - "-----BEGIN RSA PUBLIC KEY-----\n" - "MIGJAoGBANBJz8Vldl12aFeSMPLiA4nOetLDN0oxU8bB1SDhO7Uu2zdWYVYAF5J0\n" - "st7WvrVy/jA9v/fsezNAPskBanecHRSkdMTpkcgRPMHE7CTGEwIy1Yp1X4bPgDlC\n" - "VCnbs5Pcts5HnWEYNK7qHDAUn+IlmjOO+pTUY8uyq+GQVz6H9wFlAgMBAAE=\n" - "-----END RSA PUBLIC KEY-----\n" - "ntor-onion-key Gg73xH7+kTfT6bi1uNVx9gwQdQas9pROIfmc4NpAdC4=\n" - "p reject 25,119,135-139,445,563,1214,4661-4666,6346-6429,6699,6881-6999\n"; - static const char test_md_18[] = "onion-key\n" "-----BEGIN RSA PUBLIC KEY-----\n" @@ -415,16 +397,6 @@ static const char test_md_18[] = "p reject 25,119,135-139,445,563,1214,4661-4666,6346-6429,6699,6881-6999\n" "id rsa1024 Cd47okjCHD83YGzThGBDptXs9Z4\n"; -static const char test_md2_18[] = - "onion-key\n" - "-----BEGIN RSA PUBLIC KEY-----\n" - "MIGJAoGBAL2R8EfubUcahxha4u02P4VAR0llQIMwFAmrHPjzcK7apcQgDOf2ovOA\n" - "+YQnJFxlpBmCoCZC6ssCi+9G0mqo650lFuTMP5I90BdtjotfzESfTykHLiChyvhd\n" - "l0dlqclb2SU/GKem/fLRXH16aNi72CdSUu/1slKs/70ILi34QixRAgMBAAE=\n" - "-----END RSA PUBLIC KEY-----\n" - "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n" - "id rsa1024 t+J/EEITw28T5+mCkYKEXklZl6A\n"; - static const char test_md2_21[] = "onion-key\n" "-----BEGIN RSA PUBLIC KEY-----\n" @@ -444,17 +416,6 @@ test_md_generate(void *arg) ri = router_parse_entry_from_string(test_ri, NULL, 0, 0, NULL, NULL); tt_assert(ri); - md = dirvote_create_microdescriptor(ri, 8); - tt_str_op(md->body, OP_EQ, test_md_8); - - /* XXXX test family lines. */ - /* XXXX test method 14 for A lines. */ - /* XXXX test method 15 for P6 lines. */ - - microdesc_free(md); - md = NULL; - md = dirvote_create_microdescriptor(ri, 16); - tt_str_op(md->body, OP_EQ, test_md_16); microdesc_free(md); md = NULL; @@ -471,11 +432,6 @@ test_md_generate(void *arg) microdesc_free(md); md = NULL; - md = dirvote_create_microdescriptor(ri, 18); - tt_str_op(md->body, OP_EQ, test_md2_18); - - microdesc_free(md); - md = NULL; md = dirvote_create_microdescriptor(ri, 21); tt_str_op(md->body, OP_EQ, test_md2_21); tt_assert(ed25519_pubkey_eq(md->ed25519_identity_pkey, diff --git a/src/test/test_nodelist.c b/src/test/test_nodelist.c index 094e934712..7d8e57543c 100644 --- a/src/test/test_nodelist.c +++ b/src/test/test_nodelist.c @@ -7,6 +7,7 @@ **/ #include "or.h" +#include "crypto_rand.h" #include "networkstatus.h" #include "nodelist.h" #include "torcert.h" diff --git a/src/test/test_oom.c b/src/test/test_oom.c index c172fe60c7..abf8896452 100644 --- a/src/test/test_oom.c +++ b/src/test/test_oom.c @@ -13,6 +13,7 @@ #include "compat_libevent.h" #include "connection.h" #include "config.h" +#include "crypto_rand.h" #include "relay.h" #include "test.h" #include "test_helpers.h" diff --git a/src/test/test_options.c b/src/test/test_options.c index eaf5034397..65564f324c 100644 --- a/src/test/test_options.c +++ b/src/test/test_options.c @@ -2067,12 +2067,12 @@ test_options_validate__testing(void *ignored) ENSURE_DEFAULT(TestingV3AuthVotingStartOffset, 3000); ENSURE_DEFAULT(TestingAuthDirTimeToLearnReachability, 3000); ENSURE_DEFAULT(TestingEstimatedDescriptorPropagationTime, 3000); - ENSURE_DEFAULT(TestingServerDownloadSchedule, 3000); - ENSURE_DEFAULT(TestingClientDownloadSchedule, 3000); - ENSURE_DEFAULT(TestingServerConsensusDownloadSchedule, 3000); - ENSURE_DEFAULT(TestingClientConsensusDownloadSchedule, 3000); - ENSURE_DEFAULT(TestingBridgeDownloadSchedule, 3000); - ENSURE_DEFAULT(TestingBridgeBootstrapDownloadSchedule, 3000); + ENSURE_DEFAULT(TestingServerDownloadInitialDelay, 3000); + ENSURE_DEFAULT(TestingClientDownloadInitialDelay, 3000); + ENSURE_DEFAULT(TestingServerConsensusDownloadInitialDelay, 3000); + ENSURE_DEFAULT(TestingClientConsensusDownloadInitialDelay, 3000); + ENSURE_DEFAULT(TestingBridgeDownloadInitialDelay, 3000); + ENSURE_DEFAULT(TestingBridgeBootstrapDownloadInitialDelay, 3000); ENSURE_DEFAULT(TestingClientMaxIntervalWithoutRequest, 3000); ENSURE_DEFAULT(TestingDirConnectionMaxStall, 3000); ENSURE_DEFAULT(TestingAuthKeyLifetime, 3000); @@ -2422,37 +2422,6 @@ test_options_validate__circuits(void *ignored) } static void -test_options_validate__port_forwarding(void *ignored) -{ - (void)ignored; - int ret; - char *msg; - options_test_data_t *tdata = NULL; - - free_options_test_data(tdata); - tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES - "PortForwarding 1\nSandbox 1\n"); - ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg); - tt_int_op(ret, OP_EQ, -1); - tt_str_op(msg, OP_EQ, "PortForwarding is not compatible with Sandbox;" - " at most one can be set"); - tor_free(msg); - - free_options_test_data(tdata); - tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES - "PortForwarding 1\nSandbox 0\n"); - ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg); - tt_int_op(ret, OP_EQ, 0); - tt_assert(!msg); - tor_free(msg); - - done: - free_options_test_data(tdata); - policies_free_all(); - tor_free(msg); -} - -static void test_options_validate__tor2web(void *ignored) { (void)ignored; @@ -4135,16 +4104,6 @@ test_options_validate__testing_options(void *ignored) free_options_test_data(tdata); tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES "TestingEnableTbEmptyEvent 1\n" - ); - ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg); - tt_int_op(ret, OP_EQ, -1); - tt_str_op(msg, OP_EQ, "TestingEnableTbEmptyEvent may only be changed " - "in testing Tor networks!"); - tor_free(msg); - - free_options_test_data(tdata); - tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES - "TestingEnableTbEmptyEvent 1\n" VALID_DIR_AUTH "TestingTorNetwork 1\n" "___UsingTestNetworkDefaults 0\n" @@ -4261,7 +4220,6 @@ struct testcase_t options_tests[] = { LOCAL_VALIDATE_TEST(path_bias), LOCAL_VALIDATE_TEST(bandwidth), LOCAL_VALIDATE_TEST(circuits), - LOCAL_VALIDATE_TEST(port_forwarding), LOCAL_VALIDATE_TEST(tor2web), LOCAL_VALIDATE_TEST(rend), LOCAL_VALIDATE_TEST(single_onion), diff --git a/src/test/test_periodic_event.c b/src/test/test_periodic_event.c new file mode 100644 index 0000000000..f159c4f83a --- /dev/null +++ b/src/test/test_periodic_event.c @@ -0,0 +1,329 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_periodic_event.c + * \brief Test the periodic events that Tor uses for different roles. They are + * part of the libevent mainloop + */ + +#define CONFIG_PRIVATE +#define HS_SERVICE_PRIVATE +#define MAIN_PRIVATE + +#include "test.h" +#include "test_helpers.h" + +#include "or.h" +#include "config.h" +#include "hibernate.h" +#include "hs_service.h" +#include "main.h" +#include "periodic.h" + +/** Helper function: This is replaced in some tests for the event callbacks so + * we don't actually go into the code path of those callbacks. */ +static int +dumb_event_fn(time_t now, const or_options_t *options) +{ + (void) now; + (void) options; + + /* Will get rescheduled in 300 seconds. It just can't be 0. */ + return 300; +} + +static void +register_dummy_hidden_service(hs_service_t *service) +{ + memset(service, 0, sizeof(hs_service_t)); + memset(&service->keys.identity_pk, 'A', sizeof(service->keys.identity_pk)); + (void) register_service(get_hs_service_map(), service); +} + +static void +test_pe_initialize(void *arg) +{ + (void) arg; + + /* Initialize the events but the callback won't get called since we would + * need to run the main loop and then wait for a second delaying the unit + * tests. Instead, we'll test the callback work indepedently elsewhere. */ + initialize_periodic_events(); + + /* Validate that all events have been set up. */ + for (int i = 0; periodic_events[i].name; ++i) { + periodic_event_item_t *item = &periodic_events[i]; + tt_assert(item->ev); + tt_assert(item->fn); + tt_u64_op(item->last_action_time, OP_EQ, 0); + /* Every event must have role(s) assign to it. This is done statically. */ + tt_u64_op(item->roles, OP_NE, 0); + tt_uint_op(periodic_event_is_enabled(item), OP_EQ, 0); + } + + done: + teardown_periodic_events(); +} + +static void +test_pe_launch(void *arg) +{ + hs_service_t service, *to_remove = NULL; + or_options_t *options; + + (void) arg; + + hs_init(); + /* We need to put tor in hibernation live state so the events requiring + * network gets enabled. */ + consider_hibernation(time(NULL)); + + /* Hack: We'll set a dumb fn() of each events so they don't get called when + * dispatching them. We just want to test the state of the callbacks, not + * the whole code path. */ + for (int i = 0; periodic_events[i].name; ++i) { + periodic_event_item_t *item = &periodic_events[i]; + item->fn = dumb_event_fn; + } + + /* Lets make sure that before intialization, we can't scan the periodic + * events list and launch them. Lets try by being a Client. */ + options = get_options_mutable(); + options->SocksPort_set = 1; + periodic_events_on_new_options(options); + for (int i = 0; periodic_events[i].name; ++i) { + periodic_event_item_t *item = &periodic_events[i]; + tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0); + } + + initialize_periodic_events(); + + /* Now that we've initialized, rescan the list to launch. */ + periodic_events_on_new_options(options); + + for (int i = 0; periodic_events[i].name; ++i) { + periodic_event_item_t *item = &periodic_events[i]; + if (item->roles & PERIODIC_EVENT_ROLE_CLIENT) { + tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1); + } else { + tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0); + } + // enabled or not, the event has not yet been run. + tt_u64_op(item->last_action_time, OP_EQ, 0); + } + + /* Remove Client but become a Relay. */ + options->SocksPort_set = 0; + options->ORPort_set = 1; + periodic_events_on_new_options(options); + + unsigned roles = get_my_roles(options); + tt_uint_op(roles, OP_EQ, + PERIODIC_EVENT_ROLE_RELAY|PERIODIC_EVENT_ROLE_DIRSERVER); + + for (int i = 0; periodic_events[i].name; ++i) { + periodic_event_item_t *item = &periodic_events[i]; + /* Only Client role should be disabled. */ + if (item->roles == PERIODIC_EVENT_ROLE_CLIENT) { + tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0); + } + if (item->roles & PERIODIC_EVENT_ROLE_RELAY) { + tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1); + } + /* Non Relay role should be disabled, except for Dirserver. */ + if (!(item->roles & roles)) { + tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0); + } + } + + /* Disable everything and we'll enable them ALL. */ + options->SocksPort_set = 0; + options->ORPort_set = 0; + periodic_events_on_new_options(options); + + for (int i = 0; periodic_events[i].name; ++i) { + periodic_event_item_t *item = &periodic_events[i]; + tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0); + } + + /* Enable everything. */ + options->SocksPort_set = 1; options->ORPort_set = 1; + options->BridgeRelay = 1; options->AuthoritativeDir = 1; + options->V3AuthoritativeDir = 1; options->BridgeAuthoritativeDir = 1; + register_dummy_hidden_service(&service); + periodic_events_on_new_options(options); + /* Note down the reference because we need to remove this service from the + * global list before the hs_free_all() call so it doesn't try to free + * memory on the stack. Furthermore, we can't remove it now else it will + * trigger a rescan of the event disabling the HS service event. */ + to_remove = &service; + + for (int i = 0; periodic_events[i].name; ++i) { + periodic_event_item_t *item = &periodic_events[i]; + tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1); + } + + done: + if (to_remove) { + remove_service(get_hs_service_map(), to_remove); + } + hs_free_all(); +} + +static void +test_pe_get_roles(void *arg) +{ + int roles; + + (void) arg; + + /* Just so the HS global map exists. */ + hs_init(); + + or_options_t *options = get_options_mutable(); + tt_assert(options); + + /* Nothing configured, should be no roles. */ + roles = get_my_roles(options); + tt_int_op(roles, OP_EQ, 0); + + /* Indicate we have a SocksPort, roles should be come Client. */ + options->SocksPort_set = 1; + roles = get_my_roles(options); + tt_int_op(roles, OP_EQ, PERIODIC_EVENT_ROLE_CLIENT); + + /* Now, we'll add a ORPort so should now be a Relay + Client. */ + options->ORPort_set = 1; + roles = get_my_roles(options); + tt_int_op(roles, OP_EQ, + (PERIODIC_EVENT_ROLE_CLIENT | PERIODIC_EVENT_ROLE_RELAY | + PERIODIC_EVENT_ROLE_DIRSERVER)); + + /* Now add a Bridge. */ + options->BridgeRelay = 1; + roles = get_my_roles(options); + tt_int_op(roles, OP_EQ, + (PERIODIC_EVENT_ROLE_CLIENT | PERIODIC_EVENT_ROLE_RELAY | + PERIODIC_EVENT_ROLE_BRIDGE | PERIODIC_EVENT_ROLE_DIRSERVER)); + tt_assert(roles & PERIODIC_EVENT_ROLE_ROUTER); + /* Unset client so we can solely test Router role. */ + options->SocksPort_set = 0; + roles = get_my_roles(options); + tt_int_op(roles, OP_EQ, + PERIODIC_EVENT_ROLE_ROUTER | PERIODIC_EVENT_ROLE_DIRSERVER); + + /* Reset options so we can test authorities. */ + options->SocksPort_set = 0; + options->ORPort_set = 0; + options->BridgeRelay = 0; + roles = get_my_roles(options); + tt_int_op(roles, OP_EQ, 0); + + /* Now upgrade to Dirauth. */ + options->DirPort_set = 1; + options->AuthoritativeDir = 1; + options->V3AuthoritativeDir = 1; + roles = get_my_roles(options); + tt_int_op(roles, OP_EQ, + PERIODIC_EVENT_ROLE_DIRAUTH|PERIODIC_EVENT_ROLE_DIRSERVER); + tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES); + + /* Now Bridge Authority. */ + options->V3AuthoritativeDir = 0; + options->BridgeAuthoritativeDir = 1; + roles = get_my_roles(options); + tt_int_op(roles, OP_EQ, + PERIODIC_EVENT_ROLE_BRIDGEAUTH|PERIODIC_EVENT_ROLE_DIRSERVER); + tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES); + + /* Move that bridge auth to become a relay. */ + options->ORPort_set = 1; + roles = get_my_roles(options); + tt_int_op(roles, OP_EQ, + (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_RELAY + | PERIODIC_EVENT_ROLE_DIRSERVER)); + tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES); + + /* And now an Hidden service. */ + hs_service_t service; + register_dummy_hidden_service(&service); + roles = get_my_roles(options); + /* Remove it now so the hs_free_all() doesn't try to free stack memory. */ + remove_service(get_hs_service_map(), &service); + tt_int_op(roles, OP_EQ, + (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_RELAY | + PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_DIRSERVER)); + tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES); + + done: + hs_free_all(); +} + +static void +test_pe_hs_service(void *arg) +{ + hs_service_t service, *to_remove = NULL; + + (void) arg; + + hs_init(); + /* We need to put tor in hibernation live state so the events requiring + * network gets enabled. */ + consider_hibernation(time(NULL)); + /* Initialize the events so we can enable them */ + initialize_periodic_events(); + + /* Hack: We'll set a dumb fn() of each events so they don't get called when + * dispatching them. We just want to test the state of the callbacks, not + * the whole code path. */ + for (int i = 0; periodic_events[i].name; ++i) { + periodic_event_item_t *item = &periodic_events[i]; + item->fn = dumb_event_fn; + } + + /* This should trigger a rescan of the list and enable the HS service + * events. */ + register_dummy_hidden_service(&service); + /* Note down the reference because we need to remove this service from the + * global list before the hs_free_all() call so it doesn't try to free + * memory on the stack. Furthermore, we can't remove it now else it will + * trigger a rescan of the event disabling the HS service event. */ + to_remove = &service; + + for (int i = 0; periodic_events[i].name; ++i) { + periodic_event_item_t *item = &periodic_events[i]; + if (item->roles & PERIODIC_EVENT_ROLE_HS_SERVICE) { + tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1); + } + } + to_remove = NULL; + + /* Remove the service from the global map, it should trigger a rescan and + * disable the HS service events. */ + remove_service(get_hs_service_map(), &service); + for (int i = 0; periodic_events[i].name; ++i) { + periodic_event_item_t *item = &periodic_events[i]; + if (item->roles & PERIODIC_EVENT_ROLE_HS_SERVICE) { + tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0); + } + } + + done: + if (to_remove) { + remove_service(get_hs_service_map(), to_remove); + } + hs_free_all(); +} + +#define PE_TEST(name) \ + { #name, test_pe_## name , TT_FORK, NULL, NULL } + +struct testcase_t periodic_event_tests[] = { + PE_TEST(initialize), + PE_TEST(launch), + PE_TEST(get_roles), + PE_TEST(hs_service), + + END_OF_TESTCASES +}; diff --git a/src/test/test_policy.c b/src/test/test_policy.c index f8aa8ac40b..e89d49aaf5 100644 --- a/src/test/test_policy.c +++ b/src/test/test_policy.c @@ -1496,9 +1496,21 @@ test_dump_exit_policy_to_string(void *arg) } static routerinfo_t *mock_desc_routerinfo = NULL; +static int routerinfo_err; + static const routerinfo_t * -mock_router_get_my_routerinfo(void) +mock_router_get_my_routerinfo_with_err(int *err) { + if (routerinfo_err) { + if (err) + *err = routerinfo_err; + + return NULL; + } + + if (err) + *err = 0; + return mock_desc_routerinfo; } @@ -1541,7 +1553,8 @@ test_policies_getinfo_helper_policies(void *arg) tor_free(answer); memset(&mock_my_routerinfo, 0, sizeof(routerinfo_t)); - MOCK(router_get_my_routerinfo, mock_router_get_my_routerinfo); + MOCK(router_get_my_routerinfo_with_err, + mock_router_get_my_routerinfo_with_err); mock_my_routerinfo.exit_policy = smartlist_new(); mock_desc_routerinfo = &mock_my_routerinfo; @@ -1658,6 +1671,55 @@ test_policies_getinfo_helper_policies(void *arg) tt_assert(strlen(answer) == ipv4_len + ipv6_len + 1); tor_free(answer); + routerinfo_err = TOR_ROUTERINFO_ERROR_NO_EXT_ADDR; + rv = getinfo_helper_policies(NULL, "exit-policy/full", &answer, + &errmsg); + tt_int_op(rv, OP_EQ, -1); + tt_ptr_op(answer, OP_EQ, NULL); + tt_ptr_op(errmsg, OP_NE, NULL); + tt_str_op(errmsg, OP_EQ, "No known exit address yet"); + + routerinfo_err = TOR_ROUTERINFO_ERROR_CANNOT_PARSE; + rv = getinfo_helper_policies(NULL, "exit-policy/full", &answer, + &errmsg); + tt_int_op(rv, OP_EQ, -1); + tt_ptr_op(answer, OP_EQ, NULL); + tt_ptr_op(errmsg, OP_NE, NULL); + tt_str_op(errmsg, OP_EQ, "Cannot parse descriptor"); + + routerinfo_err = TOR_ROUTERINFO_ERROR_NOT_A_SERVER; + rv = getinfo_helper_policies(NULL, "exit-policy/full", &answer, + &errmsg); + tt_int_op(rv, OP_EQ, 0); + tt_ptr_op(answer, OP_EQ, NULL); + tt_ptr_op(errmsg, OP_NE, NULL); + tt_str_op(errmsg, OP_EQ, "Not running in server mode"); + + routerinfo_err = TOR_ROUTERINFO_ERROR_DIGEST_FAILED; + rv = getinfo_helper_policies(NULL, "exit-policy/full", &answer, + &errmsg); + + tt_int_op(rv, OP_EQ, 0); + tt_ptr_op(answer, OP_EQ, NULL); + tt_ptr_op(errmsg, OP_NE, NULL); + tt_str_op(errmsg, OP_EQ, "Key digest failed"); + + routerinfo_err = TOR_ROUTERINFO_ERROR_CANNOT_GENERATE; + rv = getinfo_helper_policies(NULL, "exit-policy/full", &answer, + &errmsg); + tt_int_op(rv, OP_EQ, -1); + tt_ptr_op(answer, OP_EQ, NULL); + tt_ptr_op(errmsg, OP_NE, NULL); + tt_str_op(errmsg, OP_EQ, "Cannot generate descriptor"); + + routerinfo_err = TOR_ROUTERINFO_ERROR_DESC_REBUILDING; + rv = getinfo_helper_policies(NULL, "exit-policy/full", &answer, + &errmsg); + tt_int_op(rv, OP_EQ, -1); + tt_ptr_op(answer, OP_EQ, NULL); + tt_ptr_op(errmsg, OP_NE, NULL); + tt_str_op(errmsg, OP_EQ, "Descriptor still rebuilding - not ready yet"); + done: tor_free(answer); UNMOCK(get_options); @@ -1923,11 +1985,8 @@ test_policies_fascist_firewall_allows_address(void *arg) tor_addr_port_t chosen_rs_ap; \ tor_addr_make_null(&chosen_rs_ap.addr, AF_INET); \ chosen_rs_ap.port = 0; \ - tt_int_op(fascist_firewall_choose_address_rs(&(fake_rs), \ - (fw_connection), \ - (pref_only), \ - &chosen_rs_ap), \ - OP_EQ, (expect_rv)); \ + fascist_firewall_choose_address_rs(&(fake_rs), (fw_connection), \ + (pref_only), &chosen_rs_ap); \ tt_assert(tor_addr_eq(&(expect_ap).addr, &chosen_rs_ap.addr)); \ tt_int_op((expect_ap).port, OP_EQ, chosen_rs_ap.port); \ STMT_END @@ -1940,11 +1999,8 @@ test_policies_fascist_firewall_allows_address(void *arg) tor_addr_port_t chosen_node_ap; \ tor_addr_make_null(&chosen_node_ap.addr, AF_INET); \ chosen_node_ap.port = 0; \ - tt_int_op(fascist_firewall_choose_address_node(&(fake_node), \ - (fw_connection), \ - (pref_only), \ - &chosen_node_ap), \ - OP_EQ, (expect_rv)); \ + fascist_firewall_choose_address_node(&(fake_node),(fw_connection), \ + (pref_only), &chosen_node_ap); \ tt_assert(tor_addr_eq(&(expect_ap).addr, &chosen_node_ap.addr)); \ tt_int_op((expect_ap).port, OP_EQ, chosen_node_ap.port); \ STMT_END diff --git a/src/test/test_protover.c b/src/test/test_protover.c index bdfb2d13cd..06fd575631 100644 --- a/src/test/test_protover.c +++ b/src/test/test_protover.c @@ -229,8 +229,8 @@ test_protover_vote(void *arg) /* Protocol name too long */ smartlist_clear(lst); smartlist_add(lst, (void*) "DoSaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); result = protover_compute_vote(lst, 1); tt_str_op(result, OP_EQ, ""); tor_free(result); @@ -321,10 +321,10 @@ test_protover_all_supported(void *arg) #ifndef HAVE_RUST // XXXXXX ????? tor_capture_bugs_(1); tt_assert(protover_all_supported( - "DoSaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - "aaaaaaaaaaaa=1-65536", &msg)); + "DoSaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaa=1-65536", &msg)); tor_end_capture_bugs_(); #endif diff --git a/src/test/test_relaycell.c b/src/test/test_relaycell.c index eea1f5dc80..1bd17b73bf 100644 --- a/src/test/test_relaycell.c +++ b/src/test/test_relaycell.c @@ -4,9 +4,14 @@ /* Unit tests for handling different kinds of relay cell */ #define RELAY_PRIVATE +#define CIRCUITLIST_PRIVATE #include "or.h" +#include "main.h" #include "config.h" #include "connection.h" +#include "crypto.h" +#include "circuitbuild.h" +#include "circuitlist.h" #include "connection_edge.h" #include "relay.h" #include "test.h" @@ -20,6 +25,11 @@ static uint8_t srm_answer[512]; static int srm_ttl; static time_t srm_expires; +void connection_free_minimal(connection_t*); +int connected_cell_format_payload(uint8_t *payload_out, + const tor_addr_t *addr, + uint32_t ttl); + /* Mock replacement for connection_ap_hannshake_socks_resolved() */ static void socks_resolved_mock(entry_connection_t *conn, @@ -60,6 +70,256 @@ mark_unattached_mock(entry_connection_t *conn, int endreason, (void) file; } +/* Helper: Return a newly allocated and initialized origin circuit with + * purpose and flags. A default HS identifier is set to an ed25519 + * authentication key for introduction point. */ +static origin_circuit_t * +helper_create_origin_circuit(int purpose, int flags) +{ + origin_circuit_t *circ = NULL; + + circ = origin_circuit_init(purpose, flags); + tor_assert(circ); + circ->cpath = tor_malloc_zero(sizeof(crypt_path_t)); + circ->cpath->magic = CRYPT_PATH_MAGIC; + circ->cpath->state = CPATH_STATE_OPEN; + circ->cpath->package_window = circuit_initial_package_window(); + circ->cpath->deliver_window = CIRCWINDOW_START; + circ->cpath->prev = circ->cpath; + /* Create a default HS identifier. */ + circ->hs_ident = tor_malloc_zero(sizeof(hs_ident_circuit_t)); + + return circ; +} + +static void +mock_connection_mark_unattached_ap_(entry_connection_t *conn, int endreason, + int line, const char *file) +{ + (void) line; + (void) file; + conn->edge_.end_reason = endreason; +} + +static void +mock_mark_for_close(connection_t *conn, + int line, const char *file) +{ + (void)line; + (void)file; + + conn->marked_for_close = 1; + return; +} + +static void +mock_start_reading(connection_t *conn) +{ + (void)conn; + return; +} + +static void +test_circbw_relay(void *arg) +{ + cell_t cell; + relay_header_t rh; + tor_addr_t addr; + edge_connection_t *edgeconn; + entry_connection_t *entryconn; + origin_circuit_t *circ; + int delivered = 0; + int overhead = 0; + + (void)arg; + +#define PACK_CELL(id, cmd, body_s) do { \ + memset(&cell, 0, sizeof(cell)); \ + memset(&rh, 0, sizeof(rh)); \ + memcpy(cell.payload+RELAY_HEADER_SIZE, (body_s), sizeof((body_s))-1); \ + rh.length = sizeof((body_s))-1; \ + rh.command = (cmd); \ + rh.stream_id = (id); \ + relay_header_pack((uint8_t*)&cell.payload, &rh); \ + } while (0) +#define ASSERT_COUNTED_BW() do { \ + tt_int_op(circ->n_delivered_read_circ_bw, OP_EQ, delivered+rh.length); \ + tt_int_op(circ->n_overhead_read_circ_bw, OP_EQ, \ + overhead+RELAY_PAYLOAD_SIZE-rh.length); \ + delivered = circ->n_delivered_read_circ_bw; \ + overhead = circ->n_overhead_read_circ_bw; \ + } while (0) +#define ASSERT_UNCOUNTED_BW() do { \ + tt_int_op(circ->n_delivered_read_circ_bw, OP_EQ, delivered); \ + tt_int_op(circ->n_overhead_read_circ_bw, OP_EQ, overhead); \ + } while (0) + + MOCK(connection_mark_unattached_ap_, mock_connection_mark_unattached_ap_); + MOCK(connection_start_reading, mock_start_reading); + MOCK(connection_mark_for_close_internal_, mock_mark_for_close); + + entryconn = entry_connection_new(CONN_TYPE_AP, AF_INET); + edgeconn = ENTRY_TO_EDGE_CONN(entryconn); + edgeconn->base_.state = AP_CONN_STATE_CONNECT_WAIT; + edgeconn->deliver_window = 1000; + circ = helper_create_origin_circuit(CIRCUIT_PURPOSE_C_GENERAL, 0); + edgeconn->cpath_layer = circ->cpath; + circ->cpath->state = CPATH_STATE_AWAITING_KEYS; + circ->cpath->deliver_window = 1000; + + /* Stream id 0: Not counted */ + PACK_CELL(0, RELAY_COMMAND_END, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Stream id 1: Counted */ + PACK_CELL(1, RELAY_COMMAND_END, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + ASSERT_COUNTED_BW(); + + /* Properly formatted connect cell: counted */ + PACK_CELL(1, RELAY_COMMAND_CONNECTED, "Data1234"); + tor_addr_parse(&addr, "30.40.50.60"); + rh.length = connected_cell_format_payload(cell.payload+RELAY_HEADER_SIZE, + &addr, 1024); + relay_header_pack((uint8_t*)&cell.payload, &rh); \ + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + ASSERT_COUNTED_BW(); + + /* Properly formatted resolved cell in correct state: counted */ + edgeconn->base_.state = AP_CONN_STATE_RESOLVE_WAIT; + entryconn->socks_request->command = SOCKS_COMMAND_RESOLVE; + edgeconn->on_circuit = TO_CIRCUIT(circ); + PACK_CELL(1, RELAY_COMMAND_RESOLVED, + "\x04\x04\x12\x00\x00\x01\x00\x00\x02\x00"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + ASSERT_COUNTED_BW(); + + edgeconn->base_.state = AP_CONN_STATE_OPEN; + entryconn->socks_request->has_finished = 1; + + /* Connected cell after open: not counted */ + PACK_CELL(1, RELAY_COMMAND_CONNECTED, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Resolved cell after open: not counted */ + PACK_CELL(1, RELAY_COMMAND_RESOLVED, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Drop cell: not counted */ + PACK_CELL(1, RELAY_COMMAND_DROP, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Data cell on stream 0: not counted */ + PACK_CELL(1, RELAY_COMMAND_DATA, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Data cell on open connection: counted */ + ENTRY_TO_CONN(entryconn)->marked_for_close = 0; + PACK_CELL(1, RELAY_COMMAND_DATA, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + ASSERT_COUNTED_BW(); + + /* Empty Data cell on open connection: not counted */ + ENTRY_TO_CONN(entryconn)->marked_for_close = 0; + PACK_CELL(1, RELAY_COMMAND_DATA, ""); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Sendme on valid stream: counted */ + ENTRY_TO_CONN(entryconn)->outbuf_flushlen = 0; + PACK_CELL(1, RELAY_COMMAND_SENDME, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + ASSERT_COUNTED_BW(); + + /* Sendme on valid stream with full window: not counted */ + ENTRY_TO_CONN(entryconn)->outbuf_flushlen = 0; + PACK_CELL(1, RELAY_COMMAND_SENDME, "Data1234"); + edgeconn->package_window = 500; + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Sendme on unknown stream: not counted */ + ENTRY_TO_CONN(entryconn)->outbuf_flushlen = 0; + PACK_CELL(1, RELAY_COMMAND_SENDME, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Sendme on circuit with full window: not counted */ + PACK_CELL(0, RELAY_COMMAND_SENDME, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Sendme on circuit with non-full window: counted */ + PACK_CELL(0, RELAY_COMMAND_SENDME, "Data1234"); + circ->cpath->package_window = 900; + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + ASSERT_COUNTED_BW(); + + /* End cell on non-closed connection: counted */ + PACK_CELL(1, RELAY_COMMAND_END, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn, + circ->cpath); + ASSERT_COUNTED_BW(); + + /* End cell on connection that already got one: not counted */ + PACK_CELL(1, RELAY_COMMAND_END, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Invalid extended cell: not counted */ + PACK_CELL(1, RELAY_COMMAND_EXTENDED2, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Invalid extended cell: not counted */ + PACK_CELL(1, RELAY_COMMAND_EXTENDED, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* Invalid HS cell: not counted */ + PACK_CELL(1, RELAY_COMMAND_ESTABLISH_INTRO, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_UNCOUNTED_BW(); + + /* "Valid" HS cell in expected state: counted */ + TO_CIRCUIT(circ)->purpose = CIRCUIT_PURPOSE_C_ESTABLISH_REND; + PACK_CELL(1, RELAY_COMMAND_RENDEZVOUS_ESTABLISHED, "Data1234"); + connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), NULL, + circ->cpath); + ASSERT_COUNTED_BW(); + + done: + UNMOCK(connection_start_reading); + UNMOCK(connection_mark_unattached_ap_); + UNMOCK(connection_mark_for_close_internal_); + circuit_free_(TO_CIRCUIT(circ)); + connection_free_minimal(ENTRY_TO_CONN(entryconn)); +} + /* Tests for connection_edge_process_resolved_cell(). The point of ..process_resolved_cell() is to handle an incoming cell @@ -244,6 +504,7 @@ test_relaycell_resolved(void *arg) struct testcase_t relaycell_tests[] = { { "resolved", test_relaycell_resolved, TT_FORK, NULL, NULL }, + { "circbw", test_circbw_relay, TT_FORK, NULL, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_relaycrypt.c b/src/test/test_relaycrypt.c new file mode 100644 index 0000000000..60bd479719 --- /dev/null +++ b/src/test/test_relaycrypt.c @@ -0,0 +1,185 @@ +/* Copyright 2001-2004 Roger Dingledine. + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "or.h" +#include "circuitbuild.h" +#define CIRCUITLIST_PRIVATE +#include "circuitlist.h" +#include "crypto_rand.h" +#include "relay.h" +#include "relay_crypto.h" +#include "test.h" + +static const char KEY_MATERIAL[3][CPATH_KEY_MATERIAL_LEN] = { + " 'My public key is in this signed x509 object', said Tom assertively.", + "'Let's chart the pedal phlanges in the tomb', said Tom cryptographically", + " 'Segmentation fault bugs don't _just happen_', said Tom seethingly.", +}; + +typedef struct testing_circuitset_t { + or_circuit_t *or_circ[3]; + origin_circuit_t *origin_circ; +} testing_circuitset_t; + +static int testing_circuitset_teardown(const struct testcase_t *testcase, + void *ptr); + +static void * +testing_circuitset_setup(const struct testcase_t *testcase) +{ + testing_circuitset_t *cs = tor_malloc_zero(sizeof(testing_circuitset_t)); + int i; + + for (i=0; i<3; ++i) { + cs->or_circ[i] = or_circuit_new(0, NULL); + tt_int_op(0, OP_EQ, + relay_crypto_init(&cs->or_circ[i]->crypto, + KEY_MATERIAL[i], sizeof(KEY_MATERIAL[i]), + 0, 0)); + } + + cs->origin_circ = origin_circuit_new(); + cs->origin_circ->base_.purpose = CIRCUIT_PURPOSE_C_GENERAL; + for (i=0; i<3; ++i) { + crypt_path_t *hop = tor_malloc_zero(sizeof(*hop)); + relay_crypto_init(&hop->crypto, KEY_MATERIAL[i], sizeof(KEY_MATERIAL[i]), + 0, 0); + hop->state = CPATH_STATE_OPEN; + onion_append_to_cpath(&cs->origin_circ->cpath, hop); + tt_ptr_op(hop, OP_EQ, cs->origin_circ->cpath->prev); + } + + return cs; + done: + testing_circuitset_teardown(testcase, cs); + return NULL; +} + +static int +testing_circuitset_teardown(const struct testcase_t *testcase, void *ptr) +{ + (void)testcase; + testing_circuitset_t *cs = ptr; + int i; + for (i=0; i<3; ++i) { + circuit_free_(TO_CIRCUIT(cs->or_circ[i])); + } + circuit_free_(TO_CIRCUIT(cs->origin_circ)); + tor_free(cs); + return 1; +} + +static const struct testcase_setup_t relaycrypt_setup = { + testing_circuitset_setup, testing_circuitset_teardown +}; + +/* Test encrypting a cell to the final hop on a circuit, decrypting it + * at each hop, and recognizing it at the other end. Then do it again + * and again as the state evolves. */ +static void +test_relaycrypt_outbound(void *arg) +{ + testing_circuitset_t *cs = arg; + tt_assert(cs); + + relay_header_t rh; + cell_t orig; + cell_t encrypted; + int i, j; + + for (i = 0; i < 50; ++i) { + crypto_rand((char *)&orig, sizeof(orig)); + + relay_header_unpack(&rh, orig.payload); + rh.recognized = 0; + memset(rh.integrity, 0, sizeof(rh.integrity)); + relay_header_pack(orig.payload, &rh); + + memcpy(&encrypted, &orig, sizeof(orig)); + + /* Encrypt the cell to the last hop */ + relay_encrypt_cell_outbound(&encrypted, cs->origin_circ, + cs->origin_circ->cpath->prev); + + for (j = 0; j < 3; ++j) { + crypt_path_t *layer_hint = NULL; + char recognized = 0; + int r = relay_decrypt_cell(TO_CIRCUIT(cs->or_circ[j]), + &encrypted, + CELL_DIRECTION_OUT, + &layer_hint, &recognized); + tt_int_op(r, OP_EQ, 0); + tt_ptr_op(layer_hint, OP_EQ, NULL); + tt_int_op(recognized != 0, OP_EQ, j == 2); + } + + tt_mem_op(orig.payload, OP_EQ, encrypted.payload, CELL_PAYLOAD_SIZE); + } + + done: + ; +} + +/* As above, but simulate inbound cells from the last hop. */ +static void +test_relaycrypt_inbound(void *arg) +{ + testing_circuitset_t *cs = arg; + tt_assert(cs); + + relay_header_t rh; + cell_t orig; + cell_t encrypted; + int i, j; + + for (i = 0; i < 50; ++i) { + crypto_rand((char *)&orig, sizeof(orig)); + + relay_header_unpack(&rh, orig.payload); + rh.recognized = 0; + memset(rh.integrity, 0, sizeof(rh.integrity)); + relay_header_pack(orig.payload, &rh); + + memcpy(&encrypted, &orig, sizeof(orig)); + + /* Encrypt the cell to the last hop */ + relay_encrypt_cell_inbound(&encrypted, cs->or_circ[2]); + + crypt_path_t *layer_hint = NULL; + char recognized = 0; + int r; + for (j = 1; j >= 0; --j) { + r = relay_decrypt_cell(TO_CIRCUIT(cs->or_circ[j]), + &encrypted, + CELL_DIRECTION_IN, + &layer_hint, &recognized); + tt_int_op(r, OP_EQ, 0); + tt_ptr_op(layer_hint, OP_EQ, NULL); + tt_int_op(recognized, OP_EQ, 0); + } + + relay_decrypt_cell(TO_CIRCUIT(cs->origin_circ), + &encrypted, + CELL_DIRECTION_IN, + &layer_hint, &recognized); + tt_int_op(r, OP_EQ, 0); + tt_int_op(recognized, OP_EQ, 1); + tt_ptr_op(layer_hint, OP_EQ, cs->origin_circ->cpath->prev); + + tt_mem_op(orig.payload, OP_EQ, encrypted.payload, CELL_PAYLOAD_SIZE); + } + done: + ; +} + +#define TEST(name) \ + { # name, test_relaycrypt_ ## name, 0, &relaycrypt_setup, NULL } + +struct testcase_t relaycrypt_tests[] = { + TEST(outbound), + TEST(inbound), + END_OF_TESTCASES +}; + diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c index c19d66ef9d..701227c1c7 100644 --- a/src/test/test_routerlist.c +++ b/src/test/test_routerlist.c @@ -18,8 +18,9 @@ #include "connection.h" #include "container.h" #include "control.h" +#include "crypto_rand.h" #include "directory.h" -#include "dirvote.h" +#include "dirauth/dirvote.h" #include "entrynodes.h" #include "hibernate.h" #include "microdesc.h" @@ -30,13 +31,13 @@ #include "routerlist.h" #include "routerset.h" #include "routerparse.h" -#include "shared_random.h" +#include "dirauth/shared_random.h" #include "statefile.h" #include "test.h" #include "test_dir_common.h" #include "log_test_helpers.h" -void construct_consensus(char **consensus_text_md); +void construct_consensus(char **consensus_text_md, time_t now); static authority_cert_t *mock_cert; @@ -135,7 +136,7 @@ test_routerlist_launch_descriptor_downloads(void *arg) } void -construct_consensus(char **consensus_text_md) +construct_consensus(char **consensus_text_md, time_t now) { networkstatus_t *vote = NULL; networkstatus_t *v1 = NULL, *v2 = NULL, *v3 = NULL; @@ -143,7 +144,6 @@ construct_consensus(char **consensus_text_md) authority_cert_t *cert1=NULL, *cert2=NULL, *cert3=NULL; crypto_pk_t *sign_skey_1=NULL, *sign_skey_2=NULL, *sign_skey_3=NULL; crypto_pk_t *sign_skey_leg=NULL; - time_t now = time(NULL); smartlist_t *votes = NULL; int n_vrs; @@ -258,7 +258,7 @@ test_router_pick_directory_server_impl(void *arg) rs = router_pick_directory_server_impl(V3_DIRINFO, (const int) 0, NULL); tt_ptr_op(rs, OP_EQ, NULL); - construct_consensus(&consensus_text_md); + construct_consensus(&consensus_text_md, now); tt_assert(consensus_text_md); con_md = networkstatus_parse_vote_from_string(consensus_text_md, NULL, NS_TYPE_CONSENSUS); @@ -452,6 +452,7 @@ test_directory_guard_fetch_with_no_dirinfo(void *arg) int retval; char *consensus_text_md = NULL; or_options_t *options = get_options_mutable(); + time_t now = time(NULL); (void) arg; @@ -495,7 +496,7 @@ test_directory_guard_fetch_with_no_dirinfo(void *arg) conn->requested_resource = tor_strdup("ns"); /* Construct a consensus */ - construct_consensus(&consensus_text_md); + construct_consensus(&consensus_text_md, now); tt_assert(consensus_text_md); /* Place the consensus in the dirconn */ @@ -506,7 +507,7 @@ test_directory_guard_fetch_with_no_dirinfo(void *arg) args.body_len = strlen(consensus_text_md); /* Update approx time so that the consensus is considered live */ - update_approx_time(time(NULL)+1010); + update_approx_time(now+1010); setup_capture_of_logs(LOG_DEBUG); @@ -598,11 +599,167 @@ test_routerlist_router_is_already_dir_fetching(void *arg) #undef TEST_ADDR_STR #undef TEST_DIR_PORT +static long mock_apparent_skew = 0; + +/** Store apparent_skew and assert that the other arguments are as + * expected. */ +static void +mock_clock_skew_warning(const connection_t *conn, long apparent_skew, + int trusted, log_domain_mask_t domain, + const char *received, const char *source) +{ + (void)conn; + mock_apparent_skew = apparent_skew; + tt_int_op(trusted, OP_EQ, 1); + tt_int_op(domain, OP_EQ, LD_GENERAL); + tt_str_op(received, OP_EQ, "microdesc flavor consensus"); + tt_str_op(source, OP_EQ, "CONSENSUS"); + done: + ; +} + +/** Do common setup for test_timely_consensus() and + * test_early_consensus(). Call networkstatus_set_current_consensus() + * on a constructed consensus and with an appropriately-modified + * approx_time. Callers expect presence or absence of appropriate log + * messages and control events. */ +static int +test_skew_common(void *arg, time_t now, unsigned long *offset) +{ + char *consensus = NULL; + int retval = 0; + + *offset = strtoul(arg, NULL, 10); + + /* Initialize the SRV subsystem */ + MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m); + mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL); + sr_init(0); + UNMOCK(get_my_v3_authority_cert); + + construct_consensus(&consensus, now); + tt_assert(consensus); + + update_approx_time(now + *offset); + + mock_apparent_skew = 0; + /* Caller will call UNMOCK() */ + MOCK(clock_skew_warning, mock_clock_skew_warning); + /* Caller will call teardown_capture_of_logs() */ + setup_capture_of_logs(LOG_WARN); + retval = networkstatus_set_current_consensus(consensus, "microdesc", 0, + NULL); + + done: + tor_free(consensus); + return retval; +} + +/** Test non-early consensus */ +static void +test_timely_consensus(void *arg) +{ + time_t now = time(NULL); + unsigned long offset = 0; + int retval = 0; + + retval = test_skew_common(arg, now, &offset); + (void)offset; + expect_no_log_msg_containing("behind the time published in the consensus"); + tt_int_op(retval, OP_EQ, 0); + tt_int_op(mock_apparent_skew, OP_EQ, 0); + done: + teardown_capture_of_logs(); + UNMOCK(clock_skew_warning); +} + +/** Test early consensus */ +static void +test_early_consensus(void *arg) +{ + time_t now = time(NULL); + unsigned long offset = 0; + int retval = 0; + + retval = test_skew_common(arg, now, &offset); + /* Can't use expect_single_log_msg() because of unrecognized authorities */ + expect_log_msg_containing("behind the time published in the consensus"); + tt_int_op(retval, OP_EQ, 0); + /* This depends on construct_consensus() setting valid_after=now+1000 */ + tt_int_op(mock_apparent_skew, OP_EQ, offset - 1000); + done: + teardown_capture_of_logs(); + UNMOCK(clock_skew_warning); +} + +/** Test warn_early_consensus(), expecting no warning */ +static void +test_warn_early_consensus_no(const networkstatus_t *c, time_t now, + long offset) +{ + mock_apparent_skew = 0; + setup_capture_of_logs(LOG_WARN); + warn_early_consensus(c, "microdesc", now + offset); + expect_no_log_msg_containing("behind the time published in the consensus"); + tt_int_op(mock_apparent_skew, OP_EQ, 0); + done: + teardown_capture_of_logs(); +} + +/** Test warn_early_consensus(), expecting a warning */ +static void +test_warn_early_consensus_yes(const networkstatus_t *c, time_t now, + long offset) +{ + mock_apparent_skew = 0; + setup_capture_of_logs(LOG_WARN); + warn_early_consensus(c, "microdesc", now + offset); + /* Can't use expect_single_log_msg() because of unrecognized authorities */ + expect_log_msg_containing("behind the time published in the consensus"); + tt_int_op(mock_apparent_skew, OP_EQ, offset); + done: + teardown_capture_of_logs(); +} + +/** + * Test warn_early_consensus() directly, checking both the non-warning + * case (consensus is not early) and the warning case (consensus is + * early). Depends on EARLY_CONSENSUS_NOTICE_SKEW=60. + */ +static void +test_warn_early_consensus(void *arg) +{ + networkstatus_t *c = NULL; + time_t now = time(NULL); + + (void)arg; + c = tor_malloc_zero(sizeof *c); + c->valid_after = now; + c->dist_seconds = 300; + mock_apparent_skew = 0; + MOCK(clock_skew_warning, mock_clock_skew_warning); + test_warn_early_consensus_no(c, now, 60); + test_warn_early_consensus_no(c, now, 0); + test_warn_early_consensus_no(c, now, -60); + test_warn_early_consensus_no(c, now, -360); + test_warn_early_consensus_yes(c, now, -361); + test_warn_early_consensus_yes(c, now, -600); + UNMOCK(clock_skew_warning); + tor_free(c); +} + #define NODE(name, flags) \ { #name, test_routerlist_##name, (flags), NULL, NULL } #define ROUTER(name,flags) \ { #name, test_router_##name, (flags), NULL, NULL } +#define TIMELY(name, arg) \ + { name, test_timely_consensus, TT_FORK, &passthrough_setup, \ + (char *)(arg) } +#define EARLY(name, arg) \ + { name, test_early_consensus, TT_FORK, &passthrough_setup, \ + (char *)(arg) } + struct testcase_t routerlist_tests[] = { NODE(initiate_descriptor_downloads, 0), NODE(launch_descriptor_downloads, 0), @@ -610,6 +767,13 @@ struct testcase_t routerlist_tests[] = { ROUTER(pick_directory_server_impl, TT_FORK), { "directory_guard_fetch_with_no_dirinfo", test_directory_guard_fetch_with_no_dirinfo, TT_FORK, NULL, NULL }, + /* These depend on construct_consensus() setting + * valid_after=now+1000 and dist_seconds=250 */ + TIMELY("timely_consensus1", "1010"), + TIMELY("timely_consensus2", "1000"), + TIMELY("timely_consensus3", "690"), + EARLY("early_consensus1", "689"), + { "warn_early_consensus", test_warn_early_consensus, 0, NULL, NULL }, END_OF_TESTCASES }; diff --git a/src/test/test_scheduler.c b/src/test/test_scheduler.c index ebba71266c..841fc69456 100644 --- a/src/test/test_scheduler.c +++ b/src/test/test_scheduler.c @@ -4,7 +4,6 @@ #include "orconfig.h" #include <math.h> -#include <event2/event.h> #define SCHEDULER_KIST_PRIVATE #define TOR_CHANNEL_INTERNAL_ @@ -101,62 +100,6 @@ mock_kist_networkstatus_get_param( return 12; } -/* Event base for scheduelr tests */ -static struct event_base *mock_event_base = NULL; -/* Setup for mock event stuff */ -static void mock_event_free_all(void); -static void mock_event_init(void); -static void -mock_event_free_all(void) -{ - tt_ptr_op(mock_event_base, OP_NE, NULL); - - if (mock_event_base) { - event_base_free(mock_event_base); - mock_event_base = NULL; - } - - tt_ptr_op(mock_event_base, OP_EQ, NULL); - - done: - return; -} - -static void -mock_event_init(void) -{ - struct event_config *cfg = NULL; - - tt_ptr_op(mock_event_base, OP_EQ, NULL); - - /* - * Really cut down from tor_libevent_initialize of - * src/common/compat_libevent.c to kill config dependencies - */ - - if (!mock_event_base) { - cfg = event_config_new(); -#if LIBEVENT_VERSION_NUMBER >= V(2,0,9) - /* We can enable changelist support with epoll, since we don't give - * Libevent any dup'd fds. This lets us avoid some syscalls. */ - event_config_set_flag(cfg, EVENT_BASE_FLAG_EPOLL_USE_CHANGELIST); -#endif - mock_event_base = event_base_new_with_config(cfg); - event_config_free(cfg); - } - - tt_ptr_op(mock_event_base, OP_NE, NULL); - - done: - return; -} - -static struct event_base * -tor_libevent_get_base_mock(void) -{ - return mock_event_base; -} - static int scheduler_compare_channels_mock(const void *c1_v, const void *c2_v) @@ -417,9 +360,7 @@ perform_channel_state_tests(int KISTSchedRunInterval, int sched_type) mocked_options.KISTSchedRunInterval = KISTSchedRunInterval; set_scheduler_options(sched_type); - /* Set up libevent and scheduler */ - mock_event_init(); - MOCK(tor_libevent_get_base, tor_libevent_get_base_mock); + /* Set up scheduler */ scheduler_init(); /* * Install the compare channels mock so we can test @@ -523,14 +464,12 @@ perform_channel_state_tests(int KISTSchedRunInterval, int sched_type) channel_free_all(); scheduler_free_all(); - mock_event_free_all(); done: tor_free(ch1); tor_free(ch2); UNMOCK(scheduler_compare_channels); - UNMOCK(tor_libevent_get_base); UNMOCK(get_options); cleanup_scheduler_options(); @@ -635,10 +574,7 @@ test_scheduler_loop_vanilla(void *arg) set_scheduler_options(SCHEDULER_VANILLA); mocked_options.KISTSchedRunInterval = 0; - /* Set up libevent and scheduler */ - - mock_event_init(); - MOCK(tor_libevent_get_base, tor_libevent_get_base_mock); + /* Set up scheduler */ scheduler_init(); /* * Install the compare channels mock so we can test @@ -786,7 +722,6 @@ test_scheduler_loop_vanilla(void *arg) channel_flush_some_cells_mock_free_all(); channel_free_all(); scheduler_free_all(); - mock_event_free_all(); done: tor_free(ch1); @@ -795,7 +730,6 @@ test_scheduler_loop_vanilla(void *arg) UNMOCK(channel_flush_some_cells); UNMOCK(scheduler_compare_channels); - UNMOCK(tor_libevent_get_base); UNMOCK(get_options); } @@ -917,8 +851,6 @@ test_scheduler_initfree(void *arg) tt_ptr_op(channels_pending, ==, NULL); tt_ptr_op(run_sched_ev, ==, NULL); - mock_event_init(); - MOCK(tor_libevent_get_base, tor_libevent_get_base_mock); MOCK(get_options, mock_get_options); set_scheduler_options(SCHEDULER_KIST); set_scheduler_options(SCHEDULER_KIST_LITE); @@ -935,9 +867,6 @@ test_scheduler_initfree(void *arg) scheduler_free_all(); - UNMOCK(tor_libevent_get_base); - mock_event_free_all(); - tt_ptr_op(channels_pending, ==, NULL); tt_ptr_op(run_sched_ev, ==, NULL); diff --git a/src/test/test_shared_random.c b/src/test/test_shared_random.c index 437fc38deb..2b3c8c93be 100644 --- a/src/test/test_shared_random.c +++ b/src/test/test_shared_random.c @@ -9,15 +9,18 @@ #include "or.h" #include "test.h" #include "config.h" -#include "dirvote.h" -#include "shared_random.h" -#include "shared_random_state.h" +#include "crypto_rand.h" +#include "dirauth/dirvote.h" +#include "dirauth/shared_random.h" +#include "dirauth/shared_random_state.h" +#include "log_test_helpers.h" +#include "networkstatus.h" +#include "router.h" #include "routerkeys.h" #include "routerlist.h" -#include "router.h" #include "routerparse.h" -#include "networkstatus.h" -#include "log_test_helpers.h" +#include "shared_random_client.h" +#include "voting_schedule.h" static authority_cert_t *mock_cert; @@ -170,7 +173,7 @@ test_get_state_valid_until_time(void *arg) retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:01 UTC", ¤t_time); tt_int_op(retval, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), current_time); + voting_schedule_recalculate_timing(get_options(), current_time); valid_until_time = get_state_valid_until_time(current_time); /* Compare it with the correct result */ @@ -182,7 +185,7 @@ test_get_state_valid_until_time(void *arg) retval = parse_rfc1123_time("Mon, 20 Apr 2015 19:22:00 UTC", ¤t_time); tt_int_op(retval, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), current_time); + voting_schedule_recalculate_timing(get_options(), current_time); valid_until_time = get_state_valid_until_time(current_time); format_iso_time(tbuf, valid_until_time); @@ -193,7 +196,7 @@ test_get_state_valid_until_time(void *arg) retval = parse_rfc1123_time("Mon, 20 Apr 2015 23:59:00 UTC", ¤t_time); tt_int_op(retval, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), current_time); + voting_schedule_recalculate_timing(get_options(), current_time); valid_until_time = get_state_valid_until_time(current_time); format_iso_time(tbuf, valid_until_time); @@ -204,7 +207,7 @@ test_get_state_valid_until_time(void *arg) retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:00 UTC", ¤t_time); tt_int_op(retval, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), current_time); + voting_schedule_recalculate_timing(get_options(), current_time); valid_until_time = get_state_valid_until_time(current_time); format_iso_time(tbuf, valid_until_time); @@ -242,7 +245,7 @@ test_get_start_time_of_current_run(void *arg) retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:01 UTC", ¤t_time); tt_int_op(retval, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), current_time); + voting_schedule_recalculate_timing(get_options(), current_time); run_start_time = sr_state_get_start_time_of_current_protocol_run(current_time); @@ -255,7 +258,7 @@ test_get_start_time_of_current_run(void *arg) retval = parse_rfc1123_time("Mon, 20 Apr 2015 23:59:59 UTC", ¤t_time); tt_int_op(retval, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), current_time); + voting_schedule_recalculate_timing(get_options(), current_time); run_start_time = sr_state_get_start_time_of_current_protocol_run(current_time); @@ -268,7 +271,7 @@ test_get_start_time_of_current_run(void *arg) retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:00 UTC", ¤t_time); tt_int_op(retval, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), current_time); + voting_schedule_recalculate_timing(get_options(), current_time); run_start_time = sr_state_get_start_time_of_current_protocol_run(current_time); @@ -291,7 +294,7 @@ test_get_start_time_of_current_run(void *arg) retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:15:32 UTC", ¤t_time); tt_int_op(retval, OP_EQ, 0); - dirvote_recalculate_timing(get_options(), current_time); + voting_schedule_recalculate_timing(get_options(), current_time); run_start_time = sr_state_get_start_time_of_current_protocol_run(current_time); @@ -324,7 +327,7 @@ test_get_start_time_functions(void *arg) tt_int_op(retval, OP_EQ, 0); time_t now = mock_consensus.valid_after; - dirvote_recalculate_timing(get_options(), now); + voting_schedule_recalculate_timing(get_options(), now); time_t start_time_of_protocol_run = sr_state_get_start_time_of_current_protocol_run(now); tt_assert(start_time_of_protocol_run); diff --git a/src/test/test_storagedir.c b/src/test/test_storagedir.c index a27074c21f..26606f9b6e 100644 --- a/src/test/test_storagedir.c +++ b/src/test/test_storagedir.c @@ -2,6 +2,7 @@ /* See LICENSE for licensing information */ #include "or.h" +#include "crypto_rand.h" #include "storagedir.h" #include "test.h" diff --git a/src/test/test_tortls.c b/src/test/test_tortls.c index a661eb5c5d..5b5e69b002 100644 --- a/src/test/test_tortls.c +++ b/src/test/test_tortls.c @@ -203,6 +203,17 @@ test_tortls_tor_tls_get_error(void *data) } static void +library_init(void) +{ +#ifdef OPENSSL_1_1_API + OPENSSL_init_ssl(OPENSSL_INIT_LOAD_SSL_STRINGS, NULL); +#else + SSL_library_init(); + SSL_load_error_strings(); +#endif +} + +static void test_tortls_get_state_description(void *ignored) { (void)ignored; @@ -210,9 +221,7 @@ test_tortls_get_state_description(void *ignored) char *buf; SSL_CTX *ctx; - SSL_library_init(); - SSL_load_error_strings(); - + library_init(); ctx = SSL_CTX_new(SSLv23_method()); buf = tor_malloc_zero(1000); @@ -274,8 +283,7 @@ test_tortls_get_by_ssl(void *ignored) SSL_CTX *ctx; SSL *ssl; - SSL_library_init(); - SSL_load_error_strings(); + library_init(); tor_tls_allocate_tor_tls_object_ex_data_index(); ctx = SSL_CTX_new(SSLv23_method()); @@ -322,8 +330,7 @@ test_tortls_log_one_error(void *ignored) SSL_CTX *ctx; SSL *ssl = NULL; - SSL_library_init(); - SSL_load_error_strings(); + library_init(); ctx = SSL_CTX_new(SSLv23_method()); tls = tor_malloc_zero(sizeof(tor_tls_t)); @@ -415,8 +422,7 @@ test_tortls_get_error(void *ignored) int ret; SSL_CTX *ctx; - SSL_library_init(); - SSL_load_error_strings(); + library_init(); ctx = SSL_CTX_new(SSLv23_method()); setup_capture_of_logs(LOG_INFO); @@ -516,7 +522,7 @@ test_tortls_x509_cert_free(void *ignored) tor_x509_cert_free(cert); cert = tor_malloc_zero(sizeof(tor_x509_cert_t)); - cert->cert = tor_malloc_zero(sizeof(X509)); + cert->cert = X509_new(); cert->encoded = tor_malloc_zero(1); tor_x509_cert_free(cert); } @@ -547,6 +553,15 @@ test_tortls_x509_cert_get_id_digests(void *ignored) } #ifndef OPENSSL_OPAQUE +/* + * Use only for the matching fake_x509_free() call + */ +static X509 * +fake_x509_malloc(void) +{ + return tor_malloc_zero(sizeof(X509)); +} + static void fake_x509_free(X509 *cert) { @@ -643,7 +658,7 @@ test_tortls_cert_get_key(void *ignored) crypto_pk_t *res = NULL; cert = tor_malloc_zero(sizeof(tor_x509_cert_t)); X509 *key = NULL; - key = tor_malloc_zero(sizeof(X509)); + key = fake_x509_malloc(); key->references = 1; res = tor_tls_cert_get_key(cert); @@ -793,8 +808,8 @@ test_tortls_classify_client_ciphers(void *ignored) STACK_OF(SSL_CIPHER) *ciphers; SSL_CIPHER *tmp_cipher; - SSL_library_init(); - SSL_load_error_strings(); + library_init(); + tor_tls_allocate_tor_tls_object_ex_data_index(); tls = tor_malloc_zero(sizeof(tor_tls_t)); @@ -900,8 +915,7 @@ test_tortls_client_is_using_v2_ciphers(void *ignored) SSL_SESSION *sess; STACK_OF(SSL_CIPHER) *ciphers; - SSL_library_init(); - SSL_load_error_strings(); + library_init(); ctx = SSL_CTX_new(TLSv1_method()); ssl = SSL_new(ctx); @@ -1545,8 +1559,8 @@ test_tortls_session_secret_cb(void *ignored) STACK_OF(SSL_CIPHER) *ciphers = NULL; SSL_CIPHER *one; - SSL_library_init(); - SSL_load_error_strings(); + library_init(); + tor_tls_allocate_tor_tls_object_ex_data_index(); tls = tor_malloc_zero(sizeof(tor_tls_t)); @@ -1737,8 +1751,7 @@ test_tortls_find_cipher_by_id(void *ignored) fixed_cipher2 = tor_malloc_zero(sizeof(SSL_CIPHER)); fixed_cipher2->id = 0xC00A; - SSL_library_init(); - SSL_load_error_strings(); + library_init(); ctx = SSL_CTX_new(m); ssl = SSL_new(ctx); @@ -1829,8 +1842,7 @@ test_tortls_server_info_callback(void *ignored) SSL_CTX *ctx; SSL *ssl; - SSL_library_init(); - SSL_load_error_strings(); + library_init(); ctx = SSL_CTX_new(TLSv1_method()); ssl = SSL_new(ctx); @@ -2473,8 +2485,8 @@ test_tortls_context_new(void *ignored) fixed_crypto_pk_generate_key_with_bits_result[1] = 0; fixed_tor_tls_create_certificate_result_index = 0; fixed_tor_tls_create_certificate_result[0] = NULL; - fixed_tor_tls_create_certificate_result[1] = tor_malloc_zero(sizeof(X509)); - fixed_tor_tls_create_certificate_result[2] = tor_malloc_zero(sizeof(X509)); + fixed_tor_tls_create_certificate_result[1] = X509_new(); + fixed_tor_tls_create_certificate_result[2] = X509_new(); ret = tor_tls_context_new(NULL, 0, 0, 0); tt_assert(!ret); @@ -2484,9 +2496,9 @@ test_tortls_context_new(void *ignored) fixed_crypto_pk_new_result[2] = NULL; fixed_crypto_pk_generate_key_with_bits_result_index = 0; fixed_tor_tls_create_certificate_result_index = 0; - fixed_tor_tls_create_certificate_result[0] = tor_malloc_zero(sizeof(X509)); + fixed_tor_tls_create_certificate_result[0] = X509_new(); fixed_tor_tls_create_certificate_result[1] = NULL; - fixed_tor_tls_create_certificate_result[2] = tor_malloc_zero(sizeof(X509)); + fixed_tor_tls_create_certificate_result[2] = X509_new(); ret = tor_tls_context_new(NULL, 0, 0, 0); tt_assert(!ret); @@ -2496,8 +2508,8 @@ test_tortls_context_new(void *ignored) fixed_crypto_pk_new_result[2] = NULL; fixed_crypto_pk_generate_key_with_bits_result_index = 0; fixed_tor_tls_create_certificate_result_index = 0; - fixed_tor_tls_create_certificate_result[0] = tor_malloc_zero(sizeof(X509)); - fixed_tor_tls_create_certificate_result[1] = tor_malloc_zero(sizeof(X509)); + fixed_tor_tls_create_certificate_result[0] = X509_new(); + fixed_tor_tls_create_certificate_result[1] = X509_new(); fixed_tor_tls_create_certificate_result[2] = NULL; ret = tor_tls_context_new(NULL, 0, 0, 0); tt_assert(!ret); @@ -2509,9 +2521,9 @@ test_tortls_context_new(void *ignored) fixed_crypto_pk_new_result[2] = NULL; fixed_crypto_pk_generate_key_with_bits_result_index = 0; fixed_tor_tls_create_certificate_result_index = 0; - fixed_tor_tls_create_certificate_result[0] = tor_malloc_zero(sizeof(X509)); - fixed_tor_tls_create_certificate_result[1] = tor_malloc_zero(sizeof(X509)); - fixed_tor_tls_create_certificate_result[2] = tor_malloc_zero(sizeof(X509)); + fixed_tor_tls_create_certificate_result[0] = X509_new(); + fixed_tor_tls_create_certificate_result[1] = X509_new(); + fixed_tor_tls_create_certificate_result[2] = X509_new(); fixed_tor_x509_cert_new_result_index = 0; fixed_tor_x509_cert_new_result[0] = NULL; fixed_tor_x509_cert_new_result[1] = NULL; @@ -2525,9 +2537,9 @@ test_tortls_context_new(void *ignored) fixed_crypto_pk_new_result[2] = NULL; fixed_crypto_pk_generate_key_with_bits_result_index = 0; fixed_tor_tls_create_certificate_result_index = 0; - fixed_tor_tls_create_certificate_result[0] = tor_malloc_zero(sizeof(X509)); - fixed_tor_tls_create_certificate_result[1] = tor_malloc_zero(sizeof(X509)); - fixed_tor_tls_create_certificate_result[2] = tor_malloc_zero(sizeof(X509)); + fixed_tor_tls_create_certificate_result[0] = X509_new(); + fixed_tor_tls_create_certificate_result[1] = X509_new(); + fixed_tor_tls_create_certificate_result[2] = X509_new(); fixed_tor_x509_cert_new_result_index = 0; fixed_tor_x509_cert_new_result[0] = tor_malloc_zero(sizeof(tor_x509_cert_t)); fixed_tor_x509_cert_new_result[1] = NULL; @@ -2541,9 +2553,9 @@ test_tortls_context_new(void *ignored) fixed_crypto_pk_new_result[2] = NULL; fixed_crypto_pk_generate_key_with_bits_result_index = 0; fixed_tor_tls_create_certificate_result_index = 0; - fixed_tor_tls_create_certificate_result[0] = tor_malloc_zero(sizeof(X509)); - fixed_tor_tls_create_certificate_result[1] = tor_malloc_zero(sizeof(X509)); - fixed_tor_tls_create_certificate_result[2] = tor_malloc_zero(sizeof(X509)); + fixed_tor_tls_create_certificate_result[0] = X509_new(); + fixed_tor_tls_create_certificate_result[1] = X509_new(); + fixed_tor_tls_create_certificate_result[2] = X509_new(); fixed_tor_x509_cert_new_result_index = 0; fixed_tor_x509_cert_new_result[0] = tor_malloc_zero(sizeof(tor_x509_cert_t)); fixed_tor_x509_cert_new_result[1] = tor_malloc_zero(sizeof(tor_x509_cert_t)); @@ -2557,9 +2569,9 @@ test_tortls_context_new(void *ignored) fixed_crypto_pk_new_result[2] = NULL; fixed_crypto_pk_generate_key_with_bits_result_index = 0; fixed_tor_tls_create_certificate_result_index = 0; - fixed_tor_tls_create_certificate_result[0] = tor_malloc_zero(sizeof(X509)); - fixed_tor_tls_create_certificate_result[1] = tor_malloc_zero(sizeof(X509)); - fixed_tor_tls_create_certificate_result[2] = tor_malloc_zero(sizeof(X509)); + fixed_tor_tls_create_certificate_result[0] = X509_new(); + fixed_tor_tls_create_certificate_result[1] = X509_new(); + fixed_tor_tls_create_certificate_result[2] = X509_new(); fixed_tor_x509_cert_new_result_index = 0; fixed_tor_x509_cert_new_result[0] = tor_malloc_zero(sizeof(tor_x509_cert_t)); fixed_tor_x509_cert_new_result[1] = tor_malloc_zero(sizeof(tor_x509_cert_t)); diff --git a/src/test/test_util.c b/src/test/test_util.c index 036f739b89..ec11bfd5f5 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -12,10 +12,12 @@ #include "buffers.h" #include "config.h" #include "control.h" +#include "crypto_rand.h" #include "test.h" #include "memarea.h" #include "util_process.h" #include "log_test_helpers.h" +#include "compress_zstd.h" #ifdef HAVE_PWD_H #include <pwd.h> @@ -2396,6 +2398,37 @@ test_util_compress_stream_impl(compress_method_t method, tor_free(buf3); } +/** Setup function for compression tests: handles x-zstd:nostatic + */ +static void * +compression_test_setup(const struct testcase_t *testcase) +{ + tor_assert(testcase->setup_data); + tor_assert(testcase->setup_data != (void*)TT_SKIP); + const char *methodname = testcase->setup_data; + + if (!strcmp(methodname, "x-zstd:nostatic")) { + methodname = "x-zstd"; + tor_zstd_set_static_apis_disabled_for_testing(1); + } + + return (void *)methodname; +} + +/** Cleanup for compression tests: disables nostatic */ +static int +compression_test_cleanup(const struct testcase_t *testcase, void *ptr) +{ + (void)testcase; + (void)ptr; + tor_zstd_set_static_apis_disabled_for_testing(0); + return 1; +} + +static const struct testcase_setup_t compress_setup = { + compression_test_setup, compression_test_cleanup +}; + /** Run unit tests for compression functions */ static void test_util_compress(void *arg) @@ -5875,6 +5908,13 @@ test_util_monotonic_time(void *arg) tt_u64_op(coarse_stamp_diff, OP_GE, 120); tt_u64_op(coarse_stamp_diff, OP_LE, 1200); + { + uint64_t units = monotime_msec_to_approx_coarse_stamp_units(5000); + uint64_t ms = monotime_coarse_stamp_units_to_approx_msec(units); + tt_u64_op(ms, OP_GE, 4950); + tt_u64_op(ms, OP_LT, 5050); + } + done: ; } @@ -5996,6 +6036,9 @@ test_util_monotonic_time_add_msec(void *arg) monotime_coarse_add_msec(&ct2, &ct1, 1337); tt_i64_op(monotime_diff_msec(&t1, &t2), OP_EQ, 1337); tt_i64_op(monotime_coarse_diff_msec(&ct1, &ct2), OP_EQ, 1337); + // The 32-bit variant must be within 1% of the regular one. + tt_int_op(monotime_coarse_diff_msec32_(&ct1, &ct2), OP_GT, 1323); + tt_int_op(monotime_coarse_diff_msec32_(&ct1, &ct2), OP_LT, 1350); /* Add 1337 msec twice more; make sure that any second rollover issues * worked. */ @@ -6005,6 +6048,25 @@ test_util_monotonic_time_add_msec(void *arg) monotime_coarse_add_msec(&ct2, &ct2, 1337); tt_i64_op(monotime_diff_msec(&t1, &t2), OP_EQ, 1337*3); tt_i64_op(monotime_coarse_diff_msec(&ct1, &ct2), OP_EQ, 1337*3); + tt_int_op(monotime_coarse_diff_msec32_(&ct1, &ct2), OP_GT, 3970); + tt_int_op(monotime_coarse_diff_msec32_(&ct1, &ct2), OP_LT, 4051); + + done: + ; +} + +static void +test_util_nowrap_math(void *arg) +{ + (void)arg; + + tt_u64_op(0, OP_EQ, tor_add_u32_nowrap(0, 0)); + tt_u64_op(1, OP_EQ, tor_add_u32_nowrap(0, 1)); + tt_u64_op(1, OP_EQ, tor_add_u32_nowrap(1, 0)); + tt_u64_op(4, OP_EQ, tor_add_u32_nowrap(2, 2)); + tt_u64_op(UINT32_MAX, OP_EQ, tor_add_u32_nowrap(UINT32_MAX-1, 2)); + tt_u64_op(UINT32_MAX, OP_EQ, tor_add_u32_nowrap(2, UINT32_MAX-1)); + tt_u64_op(UINT32_MAX, OP_EQ, tor_add_u32_nowrap(UINT32_MAX, UINT32_MAX)); done: ; @@ -6122,22 +6184,22 @@ test_util_get_unquoted_path(void *arg) { #name, test_util_ ## name, flags, NULL, NULL } #define COMPRESS(name, identifier) \ - { "compress/" #name, test_util_compress, 0, &passthrough_setup, \ + { "compress/" #name, test_util_compress, 0, &compress_setup, \ (char*)(identifier) } #define COMPRESS_CONCAT(name, identifier) \ { "compress_concat/" #name, test_util_decompress_concatenated, 0, \ - &passthrough_setup, \ + &compress_setup, \ (char*)(identifier) } #define COMPRESS_JUNK(name, identifier) \ { "compress_junk/" #name, test_util_decompress_junk, 0, \ - &passthrough_setup, \ + &compress_setup, \ (char*)(identifier) } #define COMPRESS_DOS(name, identifier) \ { "compress_dos/" #name, test_util_decompress_dos, 0, \ - &passthrough_setup, \ + &compress_setup, \ (char*)(identifier) } #ifdef _WIN32 @@ -6168,11 +6230,13 @@ struct testcase_t util_tests[] = { COMPRESS(gzip, "gzip"), COMPRESS(lzma, "x-tor-lzma"), COMPRESS(zstd, "x-zstd"), + COMPRESS(zstd_nostatic, "x-zstd:nostatic"), COMPRESS(none, "identity"), COMPRESS_CONCAT(zlib, "deflate"), COMPRESS_CONCAT(gzip, "gzip"), COMPRESS_CONCAT(lzma, "x-tor-lzma"), COMPRESS_CONCAT(zstd, "x-zstd"), + COMPRESS_CONCAT(zstd_nostatic, "x-zstd:nostatic"), COMPRESS_CONCAT(none, "identity"), COMPRESS_JUNK(zlib, "deflate"), COMPRESS_JUNK(gzip, "gzip"), @@ -6181,6 +6245,7 @@ struct testcase_t util_tests[] = { COMPRESS_DOS(gzip, "gzip"), COMPRESS_DOS(lzma, "x-tor-lzma"), COMPRESS_DOS(zstd, "x-zstd"), + COMPRESS_DOS(zstd_nostatic, "x-zstd:nostatic"), UTIL_TEST(gzip_compression_bomb, TT_FORK), UTIL_LEGACY(datadir), UTIL_LEGACY(memarea), @@ -6201,6 +6266,7 @@ struct testcase_t util_tests[] = { UTIL_TEST(listdir, 0), UTIL_TEST(parent_dir, 0), UTIL_TEST(ftruncate, 0), + UTIL_TEST(nowrap_math, 0), UTIL_TEST(num_cpus, 0), UTIL_TEST_WIN_ONLY(load_win_lib, 0), UTIL_TEST_NO_WIN(exit_status, 0), diff --git a/src/test/test_util_format.c b/src/test/test_util_format.c index 683d5fdac1..10645fe117 100644 --- a/src/test/test_util_format.c +++ b/src/test/test_util_format.c @@ -6,6 +6,7 @@ #include "test.h" +#include "crypto_rand.h" #define UTIL_FORMAT_PRIVATE #include "util_format.h" diff --git a/src/test/test_voting_schedule.c b/src/test/test_voting_schedule.c new file mode 100644 index 0000000000..df6058b74f --- /dev/null +++ b/src/test/test_voting_schedule.c @@ -0,0 +1,64 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" + +#include "or.h" +#include "voting_schedule.h" + +#include "test.h" + +static void +test_voting_schedule_interval_start(void *arg) +{ +#define next_interval voting_schedule_get_start_of_next_interval + (void)arg; + char buf[ISO_TIME_LEN+1]; + + // Midnight UTC tonight (as I am writing this test) + const time_t midnight = 1525651200; + format_iso_time(buf, midnight); + tt_str_op(buf, OP_EQ, "2018-05-07 00:00:00"); + + /* Some simple tests with a 50-minute voting interval */ + + tt_i64_op(next_interval(midnight, 3000, 0), OP_EQ, + midnight+3000); + + tt_i64_op(next_interval(midnight+100, 3000, 0), OP_EQ, + midnight+3000); + + tt_i64_op(next_interval(midnight+3000, 3000, 0), OP_EQ, + midnight+6000); + + tt_i64_op(next_interval(midnight+3001, 3000, 0), OP_EQ, + midnight+6000); + + /* Make sure that we roll around properly at midnight */ + tt_i64_op(next_interval(midnight+83000, 3000, 0), OP_EQ, + midnight+84000); + + /* We start fresh at midnight UTC, even if there are leftover seconds. */ + tt_i64_op(next_interval(midnight+84005, 3000, 0), OP_EQ, + midnight+86400); + + /* Now try with offsets. (These are only used for test networks.) */ + tt_i64_op(next_interval(midnight, 3000, 99), OP_EQ, + midnight+99); + + tt_i64_op(next_interval(midnight+100, 3000, 99), OP_EQ, + midnight+3099); + + done: + ; +#undef next_interval +} + +#define VS(name,flags) \ + { #name, test_voting_schedule_##name, (flags), NULL, NULL } + +struct testcase_t voting_schedule_tests[] = { + VS(interval_start, 0), + END_OF_TESTCASES +}; + diff --git a/src/test/test_workqueue.c b/src/test/test_workqueue.c index 2b03173717..cc7073850c 100644 --- a/src/test/test_workqueue.c +++ b/src/test/test_workqueue.c @@ -7,12 +7,11 @@ #include "compat_threads.h" #include "onion.h" #include "workqueue.h" -#include "crypto.h" #include "crypto_curve25519.h" +#include "crypto_rand.h" #include "compat_libevent.h" #include <stdio.h> -#include <event2/event.h> #define MAX_INFLIGHT (1<<16) @@ -159,6 +158,7 @@ static tor_weak_rng_t weak_rng; static int n_sent = 0; static int rsa_sent = 0; static int ecdh_sent = 0; +static int n_received_previously = 0; static int n_received = 0; static int no_shutdown = 0; @@ -224,18 +224,24 @@ add_n_work_items(threadpool_t *tp, int n) workqueue_entry_t **to_cancel; workqueue_entry_t *ent; - to_cancel = tor_malloc(sizeof(workqueue_entry_t*) * opt_n_cancel); + // We'll choose randomly which entries to cancel. + to_cancel = tor_calloc(opt_n_cancel, sizeof(workqueue_entry_t*)); while (n_queued++ < n) { ent = add_work(tp); if (! ent) { puts("Z"); - tor_event_base_loopexit(tor_libevent_get_base(), NULL); + tor_libevent_exit_loop_after_delay(tor_libevent_get_base(), NULL); return -1; } - if (n_try_cancel < opt_n_cancel && - tor_weak_random_range(&weak_rng, n) < opt_n_cancel) { + + if (n_try_cancel < opt_n_cancel) { to_cancel[n_try_cancel++] = ent; + } else { + int p = tor_weak_random_range(&weak_rng, n_queued); + if (p < n_try_cancel) { + to_cancel[p] = ent; + } } } @@ -256,19 +262,13 @@ add_n_work_items(threadpool_t *tp, int n) static int shutting_down = 0; static void -replysock_readable_cb(tor_socket_t sock, short what, void *arg) +replysock_readable_cb(threadpool_t *tp) { - threadpool_t *tp = arg; - replyqueue_t *rq = threadpool_get_replyqueue(tp); - - int old_r = n_received; - (void) sock; - (void) what; - - replyqueue_process(rq); - if (old_r == n_received) + if (n_received_previously == n_received) return; + n_received_previously = n_received; + if (opt_verbose) { printf("%d / %d", n_received, n_sent); if (opt_n_cancel) @@ -308,7 +308,7 @@ replysock_readable_cb(tor_socket_t sock, short what, void *arg) handle_reply_shutdown, NULL); { struct timeval limit = { 2, 0 }; - tor_event_base_loopexit(tor_libevent_get_base(), &limit); + tor_libevent_exit_loop_after_delay(tor_libevent_get_base(), &limit); } } } @@ -337,7 +337,6 @@ main(int argc, char **argv) threadpool_t *tp; int i; tor_libevent_cfg evcfg; - struct event *ev; uint32_t as_flags = 0; for (i = 1; i < argc; ++i) { @@ -411,11 +410,11 @@ main(int argc, char **argv) memset(&evcfg, 0, sizeof(evcfg)); tor_libevent_initialize(&evcfg); - ev = tor_event_new(tor_libevent_get_base(), - replyqueue_get_socket(rq), EV_READ|EV_PERSIST, - replysock_readable_cb, tp); - - event_add(ev, NULL); + { + int r = threadpool_register_reply_event(tp, + replysock_readable_cb); + tor_assert(r == 0); + } #ifdef TRACK_RESPONSES handled = bitarray_init_zero(opt_n_items); @@ -433,10 +432,10 @@ main(int argc, char **argv) { struct timeval limit = { 180, 0 }; - tor_event_base_loopexit(tor_libevent_get_base(), &limit); + tor_libevent_exit_loop_after_delay(tor_libevent_get_base(), &limit); } - event_base_loop(tor_libevent_get_base(), 0); + tor_libevent_run_event_loop(tor_libevent_get_base(), 0); if (n_sent != opt_n_items || n_received+n_successful_cancel != n_sent) { printf("%d vs %d\n", n_sent, opt_n_items); diff --git a/src/test/testing_common.c b/src/test/testing_common.c index 52729147b2..4c3fe15960 100644 --- a/src/test/testing_common.c +++ b/src/test/testing_common.c @@ -8,14 +8,17 @@ * \brief Common pieces to implement unit tests. **/ +#define MAIN_PRIVATE #include "orconfig.h" #include "or.h" #include "control.h" #include "config.h" +#include "crypto_rand.h" #include "rephist.h" #include "backtrace.h" #include "test.h" #include "channelpadding.h" +#include "main.h" #include <stdio.h> #ifdef HAVE_FCNTL_H @@ -29,8 +32,6 @@ #include <dirent.h> #endif /* defined(_WIN32) */ -#include "or.h" - #ifdef USE_DMALLOC #include <dmalloc.h> #include "main.h" @@ -292,6 +293,7 @@ main(int c, const char **v) } rep_hist_init(); setup_directory(); + initialize_mainloop_events(); options_init(options); options->DataDirectory = tor_strdup(temp_dir); tor_asprintf(&options->KeyDirectory, "%s"PATH_SEPARATOR"keys", diff --git a/src/test/testing_rsakeys.c b/src/test/testing_rsakeys.c index 7a24c0ed14..94d3db328a 100644 --- a/src/test/testing_rsakeys.c +++ b/src/test/testing_rsakeys.c @@ -3,6 +3,7 @@ * Copyright (c) 2007-2017, The Tor Project, Inc. */ /* See LICENSE for licensing information */ +#include "crypto_rand.h" #include "orconfig.h" #include "or.h" #include "test.h" diff --git a/src/tools/include.am b/src/tools/include.am index 92cc3f10a2..016cf3b124 100644 --- a/src/tools/include.am +++ b/src/tools/include.am @@ -44,8 +44,6 @@ src_tools_tor_cov_gencert_LDADD = src/common/libor-testing.a \ @TOR_LIB_WS32@ @TOR_LIB_GDI@ @CURVE25519_LIBS@ endif -EXTRA_DIST += src/tools/tor-fw-helper/README - if BUILD_LIBTORRUNNER noinst_LIBRARIES += src/tools/libtorrunner.a src_tools_libtorrunner_a_SOURCES = src/tools/tor_runner.c src/or/tor_api.c diff --git a/src/tools/tor-fw-helper/README b/src/tools/tor-fw-helper/README deleted file mode 100644 index 6a1ecaa1e4..0000000000 --- a/src/tools/tor-fw-helper/README +++ /dev/null @@ -1,10 +0,0 @@ - -We no longer recommend the use of this tool. Instead, please use the -pure-Go version of tor-fw-helper available at - https://gitweb.torproject.org/tor-fw-helper.git - -Why? - -The C code here was fine, but frankly: we don't trust the underlying -libraries. They don't seem to have been written with network security -in mind, and we have very little faith in their safety. diff --git a/src/tools/tor-gencert.c b/src/tools/tor-gencert.c index fb7465c0eb..aafefdad74 100644 --- a/src/tools/tor-gencert.c +++ b/src/tools/tor-gencert.c @@ -36,10 +36,12 @@ ENABLE_GCC_WARNING(redundant-decls) #include <assert.h> #endif -#include "compat.h" #include "util.h" #include "torlog.h" #include "crypto.h" +#include "crypto_digest.h" +#include "crypto_rand.h" +#include "crypto_util.h" #include "address.h" #include "util_format.h" diff --git a/src/trunnel/include.am b/src/trunnel/include.am index ca79ff3a39..b249fb302c 100644 --- a/src/trunnel/include.am +++ b/src/trunnel/include.am @@ -41,7 +41,12 @@ TRUNNELHEADERS = \ src_trunnel_libor_trunnel_a_SOURCES = $(TRUNNELSOURCES) src_trunnel_libor_trunnel_a_CPPFLAGS = -DTRUNNEL_LOCAL_H $(AM_CPPFLAGS) +if UNITTESTS_ENABLED src_trunnel_libor_trunnel_testing_a_SOURCES = $(TRUNNELSOURCES) +else +src_trunnel_libor_trunnel_testing_a_SOURCES = +endif + src_trunnel_libor_trunnel_testing_a_CPPFLAGS = -DTRUNNEL_LOCAL_H $(AM_CPPFLAGS) $(TEST_CPPFLAGS) src_trunnel_libor_trunnel_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) diff --git a/src/trunnel/trunnel-local.h b/src/trunnel/trunnel-local.h index b7c2ab98ef..8aa6d0ddaa 100644 --- a/src/trunnel/trunnel-local.h +++ b/src/trunnel/trunnel-local.h @@ -4,7 +4,7 @@ #include "util.h" #include "compat.h" -#include "crypto.h" +#include "crypto_util.h" #define trunnel_malloc tor_malloc #define trunnel_calloc tor_calloc diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h index 72489d7021..1f1e56e76c 100644 --- a/src/win32/orconfig.h +++ b/src/win32/orconfig.h @@ -218,7 +218,7 @@ #define USING_TWOS_COMPLEMENT /* Version number of package */ -#define VERSION "0.3.3.10-dev" +#define VERSION "0.3.4.9-dev" |